林子雨编著《大数据基础编程、实验和案例教程（第2版）》教材第13章的代码

cd /home/hadoop/下载
ls

cd /usr/local
ls
sudo mkdir bigdatacase
#这里会提示你输入当前用户（本教程是hadoop用户名）的密码
#下面给hadoop用户赋予针对bigdatacase目录的各种操作权限
sudo chown -R hadoop:hadoop ./bigdatacase
cd bigdatacase
#下面创建一个dataset目录，用于保存数据集
mkdir dataset
#下面就可以解压缩user.zip文件
cd ~  //表示进入hadoop用户的目录
cd 下载
ls
unzip user.zip -d /usr/local/bigdatacase/dataset
cd /usr/local/bigdatacase/dataset
ls

head -5 raw_user.csv

cd /usr/local/bigdatacase/dataset
#下面删除raw_user中的第1行
sed -i '1d' raw_user.csv
#上面的1d表示删除第1行，同理，3d表示删除第3行，nd表示删除第n行
#下面删除small_user中的第1行
sed -i '1d' small_user.csv
#下面再用head命令去查看文件的前5行记录，就看不到字段名称这一行了
head -5 raw_user.csv
head -5 small_user.csv

cd /usr/local/bigdatacase/dataset
vim pre_deal.sh

cd /usr/local/bigdatacase/dataset
bash ./pre_deal.sh small_user.csv user_table.txt

head -10 user_table.txt

cd /usr/local/hadoop
./sbin/start-dfs.sh

jps

cd /usr/local/hadoop
./bin/hdfs dfs -mkdir -p /bigdatacase/dataset

cd /usr/local/hadoop
./bin/hdfs dfs -put /usr/local/bigdatacase/dataset/user_table.txt /bigdatacase/dataset

cd /usr/local/hadoop
./bin/hdfs dfs -cat /bigdatacase/dataset/user_table.txt | head -10

service mysql start  #可以在Linux的任何目录下执行该命令

cd /usr/local/hive
./bin/hive   #启动Hive

hive> create database dblab;
hive> use dblab;

cd /usr/local/hadoop
./bin/hdfs dfs -ls /user/hive/warehouse/dblab.db/

mysql -u root -p 

mysql> show databases; #显示所有数据库
mysql> create database dblab; #创建dblab数据库
mysql> use dblab; #使用数据库

mysql>show variables like "char%";

mysql> CREATE TABLE `dblab`.`user_action` (`id` varchar(50),`uid` varchar(50),`item_id` varchar(50),`behavior_type` varchar(10),`item_category` varchar(50), `visit_date` DATE,`province` varchar(20)) ENGINE=InnoDB DEFAULT CHARSET=utf8;

mysql> exit;

<property>
        <name>hadoop.proxyuser.hadoop.hosts</name>
        <value>*</value>
</property>
<property>
        <name>hadoop.proxyuser.hadoop.groups</name>
        <value>*</value>
</property>

cd /usr/local/hive
./bin/hive --service hiveserver2 -hiveconf hive.server2.thrift.port=10000

sudo netstat -anp|grep 10000

import java.sql.*;
import java.sql.SQLException;
 
public class HivetoMySQL {
    private static String driverName = "org.apache.hive.jdbc.HiveDriver";
    private static String driverName_mysql = "com.mysql.jdbc.Driver";
    public static void main(String[] args) throws SQLException {
        try {
            Class.forName(driverName);
        }catch (ClassNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
            System.exit(1);
        }
        Connection con1 = DriverManager.getConnection("jdbc:hive2://localhost:10000/default", "hive", "hive");//后两个参数是用户名密码
 
        if(con1 == null)
            System.out.println("连接失败");
        else {
            Statement stmt = con1.createStatement();
            String sql = "select * from dblab.user_action";
            System.out.println("Running: " + sql);
            ResultSet res = stmt.executeQuery(sql);
 
            //InsertToMysql
            try {
                Class.forName(driverName_mysql);
                Connection con2 = DriverManager.getConnection("jdbc:mysql://localhost:3306/dblab","root","root");
                String sql2 = "insert into user_action(id,uid,item_id,behavior_type,item_category,visit_date,province) values (?,?,?,?,?,?,?)";
                PreparedStatement ps = con2.prepareStatement(sql2);
                while (res.next()) {
                    ps.setString(1,res.getString(1));
                    ps.setString(2,res.getString(2));
                    ps.setString(3,res.getString(3));
                    ps.setString(4,res.getString(4));
                    ps.setString(5,res.getString(5));
                    ps.setDate(6,res.getDate(6));
                    ps.setString(7,res.getString(7));
                    ps.executeUpdate();
                }
                ps.close();
                con2.close();
                res.close();
                stmt.close();
            } catch (ClassNotFoundException e) {
                e.printStackTrace();
            }
        }
        con1.close();
    }
}

mysql -u root -p

mysql> use dblab;
mysql> select * from user_action limit 10;

cd /usr/local/hadoop
./sbin/start-all.sh

cd /usr/local/hbase
./bin/start-hbase.sh

cd /usr/local/bigdatacase/dataset
/usr/local/hadoop/bin/hdfs dfs -get /user/hive/warehouse/dblab.db/user_action .
 #将HDFS上的user_action数据复制到本地当前目录，注意'.'表示当前目录
cat ./user_action/* | head -10   #查看前10行数据
cat ./user_action/00000* > user_action.output #将00000*文件复制一份重命名为user_action.output，*表示通配符
head user_action.output  #查看user_action.output前10行

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.List; 
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes; 
public class ImportHBase extends Thread {
    public Configuration config;
    public Connection conn;
    public Table table;
    public Admin admin;
    public ImportHBase() {
        config = HBaseConfiguration.create();
//      config.set("hbase.master", "master:60000");
//      config.set("hbase.zookeeper.quorum", "master");
        try {
            conn = ConnectionFactory.createConnection(config);
            admin = conn.getAdmin();
            table = conn.getTable(TableName.valueOf("user_action"));
        } catch (IOException e) {
            e.printStackTrace();
        }
    } 
    public static void main(String[] args) throws Exception {
        if (args.length == 0) {       //第一个参数是该jar所使用的类，第二个参数是数据集所存放的路径
            throw new Exception("You must set input path!");
        }
        String fileName = args[args.length-1];  //输入的文件路径是最后一个参数
        ImportHBase test = new ImportHBase();
        test.importLocalFileToHBase(fileName);
    }
    public void importLocalFileToHBase(String fileName) {
        long st = System.currentTimeMillis();
        BufferedReader br = null;
        try {
            br = new BufferedReader(new InputStreamReader(new FileInputStream(
                    fileName)));
            String line = null;
            int count = 0;
            while ((line = br.readLine()) != null) {
                count++;
                put(line);
                if (count % 10000 == 0)
                    System.out.println(count);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally { 
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            try {
                table.close(); // must close the client
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        long en2 = System.currentTimeMillis();
        System.out.println("Total Time: " + (en2 - st) + " ms");
    }
    @SuppressWarnings("deprecation")
    public void put(String line) throws IOException {
        String[] arr = line.split("\t", -1);
        String[] column = {"id","uid","item_id","behavior_type","item_category","date","province"};
 
        if (arr.length == 7) {
            Put put = new Put(Bytes.toBytes(arr[0]));// rowkey
            for(int i=1;i<arr.length;i++){
                put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes(column[i]),Bytes.toBytes(arr[i]));
            }
            table.put(put); // put to server
        }
    }
    public void get(String rowkey, String columnFamily, String column,
            int versions) throws IOException {
        long st = System.currentTimeMillis();
        Get get = new Get(Bytes.toBytes(rowkey));
        get.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column));
        Scan scanner = new Scan(get);
        scanner.readVersions(versions);
        ResultScanner rsScanner = table.getScanner(scanner);
        for (Result result : rsScanner) {
            final List<Cell> list = result.listCells();
            for (final Cell kv : list) {
                System.out.println(Bytes.toStringBinary(kv.getValueArray()) + "\t"
                        + kv.getTimestamp()); // mid + time
            }
        }
        rsScanner.close();
        long en2 = System.currentTimeMillis();
        System.out.println("Total Time: " + (en2 - st) + " ms");
    }
}

hbase> truncate 'user_action'

/usr/local/hadoop/bin/hadoop jar /usr/local/bigdatacase/hbase/ImportHBase.jar ImportHBase /usr/local/bigdatacase/dataset/user_action.output

habse> scan 'user_action',{LIMIT=>10}  #只查询前面10行

sudo vim /etc/apt/sources.list

sudo apt-get update

sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 51716619E084DAB9

sudo apt-get install r-base

R

sudo apt-get install libmariadb-client-lgpl-dev

R

sudo apt-get install libssl-dev
sudo apt-get install libssh2-1-dev
sudo apt-get install libcurl4-openssl-dev
sudo apt-get install libxml2-dev

service mysql start

mysql -u root -p

mysql> use dblab;
mysql> select * from user_action limit 10;