Scala+spark 2nd

download Scala+spark 2nd

of 87

  • date post

    15-Apr-2017
  • Category

    Engineering

  • view

    287
  • download

    1

Embed Size (px)

Transcript of Scala+spark 2nd

PowerPoint

Scalable Language

China Mobile

1

2

Javac

Generic JavaMartin Odersky--Scala

JavaJava

Who?

3

LispErlangHaskellJava Lisp, ScalaLisp

What?JVMJava

MonadScala

4

Twitter2009RubyScalaGuardian2011JavaScalaCourseraSparkMeetupLinkedinGiltFoursquareScala

ScalaSparkScala

5

19ActorJavaScalaScalaSparkScala

SparkScala

6

12Java3 Java

7

3 12

8

Expr = Iden

| Iden => Expr

| (Expr) (Expr)

x, y, name, id, personx => name, x => id, x => xx(y), y(x), (x => name) y, (x => +(x)(1)) 3x => y => +(x)(y)

9

x => x == y => y, x => +(x)(z) == y => +(y)(z)(x => +(x)(3)) 2 == +(2)(3)(x => y => +(x)(y)) 2 3 == +(2)(3)x => f(x) == f

10

Zero = f => x => xOne = f => x => f(x)Two = f => x => f(f(x))

Succ = n => f => x => f(n(f)(x)) type ChurchNumber[A] = (A => A) => A => A def zero[A]: ChurchNumber[A] = f => a => a def succ[A](n: ChurchNumber[A]): ChurchNumber[A] = f => a => f(n(f)(a))

val a1: Int = 0 val f1: Int => Int = x => x + 1

val a2: List[Int] = List() val f2: List[Int] => List[Int] = list => 1 :: list

val a3: String = "" val f3: String => String = s => "|" + s println(zero(f1)(a1); println(succ(succ(zero))(f1)(a1)Number = Zero | Succ Number

11

type Segment = (List[Int], List[Int], List[Int])object Split { def unapply (xs: List[Int]) = { val pivot = xs(xs.size / 2) @tailrec def partition (s: Segment, ys: List[Int]): Segment = { val (left, mid, right) = s ys match { case Nil => s case head :: tail if head < pivot => partition((head :: left, mid, right), tail) case head :: tail if head == pivot => partition((left, head :: mid, right), tail) case head :: tail if head > pivot => partition((left, mid, head :: right), tail) } } Some(partition((Nil, Nil, Nil), xs)) }}

def qsort(xs: List[Int]): List[Int] = xs match { case Nil => xs case Split(left, pivot, right) => qsort(left) ::: pivot ::: qsort(right)}

Quick SortExtractorGuard

12

Pattern Matchingdef sum(list: List[Int]): Int = if (list.isEmpty) 0 else list.head + sum(list.tail)

def sum(list: List[Int]): Int = list match { case List() => result case head :: tail => head + sum(tail)}

13

def sum(list: List[Int], acc: Int): Int = list match { case Nil => result case head :: tail => sum(tail, result + head)}

14

15

var list = (1 to 100).toArrayfor (int i = 1; i Intfold0((x, y) => x + y)fold1((x, y) => x * y): ((Int, Int) => Int) => Int : ((Int, Int) => Int) => Int + : Int => Int => Int

20

class Pair[A](var x: A, var y: A) { def modifyX(x: A) = this.x = x def modifyY(y: A) = this.y = y}

var pair = new Pair(1, 2)var pair1 = new Pair(pair, pair)var pair2 = new Pair(pair, new Pair(1, 2))

pair.modifyX(3)

21

var variable = 0

implicit class FooInt(i: Int) { def |+|(j: Int) = { variable = (i + j) / 2 i + j + variable }}

(1 |+| 2) |+| 31 |+| (2 |+| 3)= 10= 12

22

var variable = 0

implicit class FooInt(i: Int) { def |+|(j: Int) = { variable += 1 i + j * variable }}

(1 |+| 2) |+| 31 |+| (2 |+| 3)= 9= 11

23

map(f: T => U): A[U]filter(f: T => Boolean): A[T]flatMap(f: T => A[T]): A[T]groupBy(f: T => K): A[(K, List[T])]sortBy(f: T => K): A[T]NEW

Count: IntForce: A[T]Reduce(f: (T, T) => T): THigher-Order Functions

TranformationAction

24

AB

Map

[A] -> (A -> B) -> [B]

List(1, 2, 3, 4).map(_.toString)(A -> B) -> ([A] -> [B])

25

AA

Filter

?A[A] -> (A -> Boolean) -> [A]

List(1, 2, 3, 4).filter(_ < 3)(A -> Boolean) -> ([A] -> [A])

26

AB

Fold

[A] -> B -> (B -> A -> B) -> [B]

val list = List(one, two, three)list.foldLeft(0)((sum, str) => { if (str.contains(o) sum + 1 else sum})B -> (B -> A -> B) -> ([A] -> [B])

27

[A]A

Flatten

[[A]] -> [A]

List(List(1, 2), List(3, 5)).flatten

28

Quick Sortobject Split { def unapply (xs: List[Int]) = { val pivot = xs(xs.size / 2) Some(xs.partitionBy(pivot)) }}

def qsort(xs: List[Int]): List[Int] = xs match { case Nil => xs case Split(left, pivot, right) => qsort(left) ::: pivot ::: qsort(right)}

29

Quick Sorttype Segment = (List[Int], List[Int], List[Int])implicit class ListWithPartition(list: List[Int]) extends AnyVal { def partitionBy(p: Int): Segment = { val idenElem = (List[Int](), List[Int](), List[Int]()) def partition(result: Segment, x: Int): Segment = { val (left, mid, right) = result if (x < p) (x :: left, mid, right) else if (x == p) (left, x :: mid, right) else (left, mid, x :: right) } list.foldLeft(idenElem)(partition) }}

30

AB

Map

[A] -> (A -> B) -> [B]Par

31

val foo = List(1, 2, 3, 4, 5)baz = foo.map(5 +).map(3 +).filter(_ > 10).map(4 *)baz.take(2)foo.map(5 +)foo.map(5 +).map(3 +)foo.map(5 +).map(3 +).filter(_ > 10)

for(int i = 0; i < 5; ++i) { int x = foo[i] + 5 + 3 if (x > 10) bar.add(x * 4) else continue;{()

32

AB

Map

[A] -> (A -> B) -> [B]View

33

val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map(n => n._1 + n._2)

Quora

zip = ([A], [B]) => [(A, B)]

34

Lazy val x = 3 + 3

def number = {println("OK"); 3 + 3}

class LazyValue(expr: => Int) { var evaluated: Boolean = false var value: Int = -1

def get: Int = { if (!evaluated) { value = expr evaluated = true } value }}

val lazyValue = new LazyValue(number)println(lazyValue.get)println(lazyValue.get)Thinking in JavaMapCall By Name

35

ScalaJava

121.1

1 + 21.+(2)

x: Int => x.toString

Function1[Int, String]map(5 +) map(+ 5)

36

class Sugar(i: Int) { def unary_- = -i def apply(expr: => Unit) = for (j = ! + -* / %

DSL

37

Mix-inInterface

1.2.Trait3.

ScalaTraitTrait & Mix-in...abstract class Bird(kind: String) { val name: String def singMyName = println(s"$name is singing") val capability: Int def run = println(s"I can run $capability meters!!!") def fly = println(s"flying of kind: $kind")}()

38

trait Runnable { val capability: Int def run = println(s"I can run $capability meters!!!")}

trait Singer { val name: String def singMyName = println(s"$name is singing")}

abstract class Bird(kind: String) { def fly = println(s"flying of kind: $kind")}

39

class Nightingale extends Bird("Nightingale") with Singer with Runnable { val capability = 20 val name = "poly"}

val myTinyBird = new NightingalemyTinyBird.flymyTinyBird.singMyNamemyTinyBird.run

class Coder(language: String) { val capability = 10 val name = "Handemelindo" def code = println(s"coding in $language")}

val me = new Coder("Scala") with Runnable with Singerme.codeme.singMyNameme.run

40

object Sugar { def apply(i: Int) = new Sugar(i)}

41

trait class Treecase class Leaf(info: String) extends Treecase class Node(left: Tree, right: Tree) extends Tree

def traverse(tree: Tree): Unit = { tree match { case Leaf(info) => println(info) case Node(left, right) => { traverse(left) traverse(right) } }}

val tree: Tree = new Node(new Node(new Leaf("1"), new Leaf("2")), new Leaf("3"))traverse(tree)Case ClassADTcase class

Tree = Leaf String | Node Tree Tree

42

C

Java

R

Ruby

ScalaUMLJavae.g. [(K1, V1)] -> [(K2, [V2])] -> [(K2, V3)]MapReduce

43

*AnyInt1Pair[Int, Int](1, 2)List[Int][1, 2, 3]*****ListPairKindTypeValue

Generics of a Higher Kind - Martin Odersky=>=>=>Proper Type

44

type Int :: *type String :: *type (Int => String) :: *type List[Int] :: *

type List :: ?type Function1 :: ??

type List :: * => *type function1 :: * => * => * Function1[-T, +R]

def id(x: Int) = xtype Id[A] = Atype id[A[_], B] = A[B]def id(f: Int => Int, x: Int) = f(x)

45

type Pair[K[_], V[_]] = (K[A], V[A]) forSome { type A }(* -> *) -> (* -> *) -> *(Set(x,x,x,x,x), List(x,x,x,x,x,x,x,x,x,x))val pair: Pair[Set, List] = (Set(42), List(52))val pair: Pair[Set, List] = (Set(42), List(52))

46

type function1 :: * => * => * def foo[A[_]](bar: A[Int]): A[Int] = bar

(* => *)val foo1 = foo[List](List(1, 2, 3, 5, 8, 13))

def baz(x: Int) = println(x)

Type Lambda * => * = *[Unit] => *[Unit] val foo2 = foo[ ({type F[X] = Function1[X, Unit]})#F ](baz)

47

trait Monoid[A]{ val zero: A def append(x: A, y: A): A}

object IntNum extends Monoid[Int] { val zero