ODPS 使用

参考地址：

https://help.aliyun.com/document_detail/27803.html

https://book.douban.com/subject/26284706/

https://bbs.aliyun.com/read/182132.html

服务配置

在阿里云机器上弹性云服务器，存储服务，数据计算服务

云服务器：这个一般会用到。不在冗余介绍。

存储服务：http://i.aliyun.com/access_key获取该key

数据计算服务：https://www.aliyun.com/product/odps 开通该服务，在https://workbench.shuju.aliyun.com/console#/ 创建项目和管理项目

客户端程序配置

下载客户端程序：https://help.aliyun.com/document_detail/27991.html

配置文件：

access_id=*******************
access_key=*********************
    # Accesss ID及Access Key是用户的云账号信息，登陆阿里云官网，进入阿里云管理控制台 —> AccessKey管理控制台查看
project_name=my_project
    # 指定用户想进入的项目空间。
end_point=https://service.odps.aliyun.com/api
    # MaxCompute服务的访问链接
tunnel_endpoint=https://dt.odps.aliyun.com
    # MaxCompute Tunnel服务的访问链接
log_view_host=http://logview.odps.aliyun.com
    # 当用户执行一个作业后，客户端会返回该作业的LogView地址。打开该地址将会看到作业执行的详细信息
https_check=true
    #决定是否开启HTTPS访问

运行程序：./bin/odpscmd

简单命令

ls tables; # 展示该项目所有表
create table table_name(key bigint) partitioned by (pt string); # 创建表和分区
alter table table_name add partition (pt = '20170101'); # 添加表和分区
insert overwrite table table_name select * from table_name where 1 > 2; # 清空表table_name
tunnel upload /root/1.txt table_name; # 上传数据到table_name，只支持单列
select count(dictinct column01, column02) from table_name; # 去重求和，其实是不规范的做饭，不能count(dictinct)
insert into table table_name partition (pt = '20170101') select dictinct column01, column02 from table_name;
select column, count(*) as count from table_name group by column01 order by column02 desc limit 999999;
insert into table table_name partition (pt = '20170101') select column01, count(*) as count from table_name_data group by column01, column02 order by count desc limit 999999;
insert into table table_name partition (pt = '20170101') select column01, count(*) as count from table_name_data where column03 = SOME and pt = '20170101' gourp by column01, column02 order by count desc limit 999999;