林子雨编著《大数据基础编程、实验和案例教程（第2版）》(教材官网)教材中的命令行和代码，在纸质教材中的印刷效果不是很好，可能会影响读者对命令行和代码的理解，为了方便读者正确理解命令行和代码或者直接拷贝命令行和代码用于上机实验，这里提供全书配套的所有命令行和代码。
查看教材所有章节的代码
第9章 Spark的安装和基础编程
教材第153页
（温馨提示：代码框上方的复制代码按钮，也就是“两张A4纸图标”，用鼠标点击复制代码按钮，就可以把代码框中的代码复制到粘贴板，粘贴到其他地方。但是，有的浏览器可能不支持该功能）
cd ~
sudo tar -zxvf ~/Downloads/spark-2.4.0-bin-without-hadoop.tgz -C /usr/local/
cd /usr/local
sudo mv ./spark-2.4.0-bin-without-hadoop/ ./spark
sudo chown -R hadoop:hadoop ./spark # hadoop是当前登录Linux系统的用户名
cd /usr/local/spark
cp ./conf/spark-env.sh.template ./conf/spark-env.sh
export SPARK_DIST_CLASSPATH=$(/usr/local/hadoop/bin/hadoop classpath)
cd /usr/local/spark
bin/run-example SparkPi
教材第154页
bin/run-example SparkPi 2>&1 | grep "Pi is roughly"
cd /usr/local/spark
./bin/spark-shell
教材第155页
scala> 8*2+5
res0: Int = 21
scala>:quit
cd /usr/local/spark
./bin/spark-shell
scala> val textFile = sc.textFile("file:///usr/local/spark/README.md")
scala> textFile.first()
cd /usr/local/hadoop
./sbin/start-dfs.sh
cd /usr/local/hadoop
./bin/hdfs dfs –put /usr/local/spark/README.md .
教材第156页
./bin/hdfs dfs -cat README.md
scala> val textFile = sc.textFile("hdfs://localhost:9000/user/hadoop/README.md")
scala> textFile.first()
scala> val textFile = sc.textFile("hdfs://localhost:9000/user/hadoop/ README.md ")
scala> val textFile = sc.textFile("/user/hadoop/ README.md ")
scala> val textFile = sc.textFile("README.md ")
scala> val textFile = sc.textFile("file:///usr/local/spark/ README.md ")
scala> val wordCount = textFile.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey((a, b) => a + b)
scala> wordCount.collect()
教材第157页
sudo mkdir /usr/local/sbt # 创建安装目录
cd ~/Downloads
sudo tar -zxvf ./sbt-1.3.8.tgz -C /usr/local
cd /usr/local/sbt
sudo chown -R hadoop /usr/local/sbt # 此处的hadoop为系统当前用户名
cp ./bin/sbt-launch.jar ./ #把bin目录下的sbt-launch.jar复制到sbt安装目录下
vim /usr/local/sbt/sbt
教材第158页
#!/bin/bash
SBT_OPTS="-Xms512M -Xmx1536M -Xss1M -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=256M"
java $SBT_OPTS -jar `dirname $0`/sbt-launch.jar "$@"
chmod u+x /usr/local/sbt/sbt
cd /usr/local/sbt
./sbt sbtVersion
Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=256M; support was removed in 8.0
[warn] No sbt.version set in project/build.properties, base directory: /usr/local/sbt
[info] Set current project to sbt (in build file:/usr/local/sbt/)
[info] 1.3.8
cd ~ # 进入用户主文件夹
mkdir ./sparkapp # 创建应用程序根目录
mkdir -p ./sparkapp/src/main/scala # 创建所需的文件夹结构
cd ~
vim ./sparkapp/src/main/scala/SimpleApp.scala
/* SimpleApp.scala */
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
object SimpleApp {
def main(args: Array[String]) {
val logFile = "file:///usr/local/spark/README.md" // Should be some file on your system
val conf = new SparkConf().setAppName("Simple Application")
val sc = new SparkContext(conf)
val logData = sc.textFile(logFile, 2).cache()
val numAs = logData.filter(line => line.contains("a")).count()
val numBs = logData.filter(line => line.contains("b")).count()
println("Lines with a: %s, Lines with b: %s".format(numAs, numBs))
}
}
教材第159页
cd ~
vim ./sparkapp/simple.sbt
name := "Simple Project"
version := "1.0"
scalaVersion := "2.11.12"
libraryDependencies += "org.apache.spark" %% "spark-core" % "2.4.0"
cd ~/sparkapp
find .
教材第160页
cd ~/sparkapp #一定把这个目录设置为当前目录
/usr/local/sbt/sbt package
~/sparkapp$ /usr/local/sbt/sbt package
/usr/local/spark/bin/spark-submit --class "SimpleApp" ~/sparkapp/target/scala-2.11/simple-project_2.11-1.0.jar
/usr/local/spark/bin/spark-submit --class "SimpleApp" ~/sparkapp/target/scala-2.11/simple-project_2.11-1.0.jar 2>&1 | grep "Lines with a:"
教材第161页
https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip
sudo unzip ~/下载/apache-maven-3.6.3-bin.zip -d /usr/local
cd /usr/local
sudo mv apache-maven-3.6.3/ ./maven
sudo chown -R hadoop ./maven
cd ~ #进入用户主文件夹
mkdir -p ./sparkapp2/src/main/java
vim ./sparkapp2/src/main/java/SimpleApp.java
/*** SimpleApp.java ***/
import org.apache.spark.api.java.*;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.SparkConf;
public class SimpleApp {
public static void main(String[] args) {
String logFile = "file:///usr/local/spark/README.md"; // Should be some file on your system
SparkConf conf=new SparkConf().setMaster("local").setAppName("SimpleApp");
JavaSparkContext sc=new JavaSparkContext(conf);
JavaRDD<String> logData = sc.textFile(logFile).cache();
long numAs = logData.filter(new Function<String, Boolean>() {
public Boolean call(String s) { return s.contains("a"); }
}).count();
long numBs = logData.filter(new Function<String, Boolean>() {
public Boolean call(String s) { return s.contains("b"); }
}).count();
System.out.println("Lines with a: " + numAs + ", lines with b: " + numBs);
}
}
教材第162页
cd ~
vim ./sparkapp2/pom.xml
<project>
<groupId>cn.edu.xmu</groupId>
<artifactId>simple-project</artifactId>
<modelVersion>4.0.0</modelVersion>
<name>Simple Project</name>
<packaging>jar</packaging>
<version>1.0</version>
<repositories>
<repository>
<id>jboss</id>
<name>JBoss Repository</name>
<url>http://repository.jboss.com/maven2/</url>
</repository>
</repositories>
<dependencies>
<dependency> <!-- Spark dependency -->
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.4.0</version>
</dependency>
</dependencies>
</project>
教材第163页
cd ~/sparkapp2
find .
cd ~/sparkapp2 #一定把这个目录设置为当前目录
/usr/local/maven/bin/mvn package
教材第164页
/usr/local/spark/bin/spark-submit --class "SimpleApp" ~/sparkapp2/target/simple-project-1.0.jar
/usr/local/spark/bin/spark-submit --class "SimpleApp" ~/sparkapp2/target/simple-project-1.0.jar 2>&1 | grep "Lines with a"