林子雨编著《大数据基础编程、实验和案例教程》教材第9章的代码

林子雨编著《大数据基础编程、实验和案例教程》(教材官网)教材中的代码，在纸质教材中的印刷效果不是很好，可能会影响读者对代码的理解，为了方便读者正确理解代码或者直接拷贝代码用于上机实验，这里提供全书配套的所有代码。
查看教材所有章节的代码

第9章 Spark的安装和基础编程

教材第170页

（温馨提示：代码框上方的复制代码按钮，也就是“两张A4纸图标”，用鼠标点击复制代码按钮，就可以把代码框中的代码复制到粘贴板，粘贴到其他地方。但是，有的浏览器可能不支持该功能）

sudo tar -zxf ~/下载/spark-1.6.2-bin-without-hadoop.tgz -C /usr/local/
cd /usr/local
sudo mv ./spark-1.6.2-bin-without-hadoop/ ./spark
sudo chown -R hadoop:hadoop ./spark    # hadoop是当前登录Linux系统的用户名

cd /usr/local/spark
cp ./conf/spark-env.sh.template ./conf/spark-env.sh

教材第171页

export SPARK_DIST_CLASSPATH=$(/usr/local/hadoop/bin/hadoop classpath)

cd /usr/local/spark
bin/run-example SparkPi

bin/run-example SparkPi 2>&1 | grep "Pi is roughly"

教材第172页

cd /usr/local/spark
./bin/spark-shell

scala> 8*2+5
res0: Int = 21

scala>:quit

cd /usr/local/spark
./bin/spark-shell

scala> val textFile = sc.textFile("file:///usr/local/spark/README.md")
scala> textFile.first()

教材第173页

cd /usr/local/hadoop
./sbin/start-dfs.sh

cd /usr/local/hadoop
./bin/hdfs dfs –put /usr/local/spark/README.md .

./bin/hdfs dfs –cat README.md

scala> val textFile = sc.textFile("hdfs://localhost:9000/user/hadoop/README.md")
scala> textFile.first()

scala>  val textFile = sc.textFile("hdfs://localhost:9000/user/hadoop/ README.md ")
scala>  val textFile = sc.textFile("/user/hadoop/ README.md ")
scala>  val textFile = sc.textFile("README.md ")

教材第174页

scala> val textFile = sc.textFile("file:///usr/local/spark/ README.md ")
scala> val wordCount = textFile.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey((a, b) => a + b)
scala> wordCount.collect()

教材第175页

https://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.11/sbt-launch.jar

sudo mkdir /usr/local/sbt
sudo chown -R hadoop /usr/local/sbt  # 此处的hadoop是Linux系统当前登录用户名
cd /usr/local/sbt
cp ~/下载/sbt-launch.jar .

vim ./sbt

#!/bin/bash
SBT_OPTS="-Xms512M -Xmx1536M -Xss1M -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=256M"
java $SBT_OPTS -jar `dirname $0`/sbt-launch.jar "$@"

chmod u+x ./sbt

./sbt sbt-version

教材第176页

cd ~           # 进入用户主文件夹
mkdir ./sparkapp        # 创建应用程序根目录
mkdir -p ./sparkapp/src/main/scala     # 创建所需的文件夹结构

cd ~
vim ./sparkapp/src/main/scala/SimpleApp.scala

/* SimpleApp.scala */
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf

object SimpleApp {
    def main(args: Array[String]) {
        val logFile = "file:///usr/local/spark/README.md" // Should be some file on your system
        val conf = new SparkConf().setAppName("Simple Application")
        val sc = new SparkContext(conf)
        val logData = sc.textFile(logFile, 2).cache()
        val numAs = logData.filter(line => line.contains("a")).count()
        val numBs = logData.filter(line => line.contains("b")).count()
        println("Lines with a: %s, Lines with b: %s".format(numAs, numBs))
    }
}

教材第177页

cd ~
vim ./sparkapp/simple.sbt

name := "Simple Project"
version := "1.0"
scalaVersion := "2.10.5"
libraryDependencies += "org.apache.spark" %% "spark-core" % "1.6.2"

cd ~/sparkapp
find .

cd ~/sparkapp  #一定把这个目录设置为当前目录
/usr/local/sbt/sbt package

教材第178页

/usr/local/spark/bin/spark-submit --class "SimpleApp" ~/sparkapp/target/scala-2.10/simple-project_2.10-1.0.jar

/usr/local/spark/bin/spark-submit --class "SimpleApp" ~/sparkapp/target/scala-2.10/simple-project_2.10-1.0.jar 2>&1 | grep "Lines with a:"

教材第179页

http://apache.fayea.com/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.zip

sudo unzip ~/下载/apache-maven-3.3.9-bin.zip -d /usr/local
cd /usr/local
sudo mv apache-maven-3.3.9/ ./maven
sudo chown -R hadoop ./maven

cd ~ #进入用户主文件夹
mkdir -p ./sparkapp2/src/main/java

vim ./sparkapp2/src/main/java/SimpleApp.java

/*** SimpleApp.java ***/
import org.apache.spark.api.java.*;
import org.apache.spark.api.java.function.Function; 
public class SimpleApp {
    public static void main(String[] args) {
        String logFile = "file:///usr/local/spark/README.md"; // Should be some file on your system
        JavaSparkContext sc = new JavaSparkContext("local", "Simple App",
            "file:///usr/local/spark/", new String[]{"target/simple-project-1.0.jar"});
        JavaRDD<String> logData = sc.textFile(logFile).cache(); 
        long numAs = logData.filter(new Function<String, Boolean>() {
            public Boolean call(String s) { return s.contains("a"); }
        }).count(); 
        long numBs = logData.filter(new Function<String, Boolean>() {
            public Boolean call(String s) { return s.contains("b"); }
        }).count(); 
        System.out.println("Lines with a: " + numAs + ", lines with b: " + numBs);
    }
}

教材第180页

cd ~
vim ./sparkapp2/pom.xml

<project>
    <groupId>edu.berkeley</groupId>
    <artifactId>simple-project</artifactId>
    <modelVersion>4.0.0</modelVersion>
    <name>Simple Project</name>
    <packaging>jar</packaging>
    <version>1.0</version>
    <repositories>
        <repository>
            <id>Akka repository</id>
            <url>http://repo.akka.io/releases</url>
        </repository>
    </repositories>
    <dependencies>
        <dependency> <!-- Spark dependency -->
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.10</artifactId>
            <version>1.6.2</version>
        </dependency>
    </dependencies>
</project>

教材第181页

cd ~/sparkapp2
find .

cd ~/sparkapp2    #一定把这个目录设置为当前目录
/usr/local/maven/bin/mvn package

/usr/local/spark/bin/spark-submit --class "SimpleApp" ~/sparkapp2/target/simple-project-1.0.jar

/usr/local/spark/bin/spark-submit --class "SimpleApp" ~/sparkapp2/target/simple-project-1.0.jar 2>&1 | grep "Lines with a"