hadoop hive udaf

hive 复杂 UDAF 使用方法

abloz 2013-03-27
2013-03-27

周海汉 /文 2013.3.27

前文《hive mapreduce script用法示例》 示例了mapreduce脚本。本文采用较复杂的方式自定义hive聚合函数。

package com.abloz.hive;
/**
 * @author zhouhh
 * @date 2013-3-27
 * note: for count value >=1
 */
import org.apache.hadoop.hive.ql.exec.UDAF;
import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;



public class JJPokerStat extends UDAF {
	public static class JJCountUDAFEvaluator implements UDAFEvaluator {
		public static final Logger log = Logger.getLogger(JJPokerStat.class);
		public static class PartialResult {
			int total=0;
			int win=0;
			int fold=0;
			int allin=0;
		}
		private PartialResult result;

		public void init() {
			result = new PartialResult();
		}

		public boolean calwin(int rbet,String chipwon)
		{
			if(chipwon.equalsIgnoreCase("NULL"))
			{
				return false;
			}

			String[] cw = chipwon.split("\|");
			int chipwons=0;
			//log.info("calwin:"+chipwon);
			for(String v:cw)
			{

				String[] c = v.split(":");
				//log.info("calwin:v "+v+",c.length:"+c.length);
				if(c.length>1)
				{
					chipwons += Integer.parseInt(c[1]);
				}
			}
			//log.info("calwin:chipwons:"+chipwons+",rbet:"+rbet);
			if(chipwons>rbet)
			{
				return true;
			}
			return false;
		}


		public boolean iterate(IntWritable rbet,Text chipwon,IntWritable f,IntWritable a) {
			if ( rbet == null || chipwon == null || f == null || a == null) {
				return true;
			}
			boolean win = calwin(rbet.get(),chipwon.toString());

			if(result == null)
			{
				result = new PartialResult();
			}

			result.total++;

			if(win)
			{
				result.win++;
			}


			int v = f.get();
			if (v>=1)
			{
				result.fold++;
			}
			v = a.get();
			if (v>=1)
			{
				result.allin++;
			}


			return true;
		}

		public PartialResult terminatePartial() {
			return result;
		}

		public boolean merge(PartialResult other) {
			if(other == null)
			{
				return true;
			}
			result.total+=other.total;
			result.win += other.win;
			result.fold += other.fold;
			result.allin += other.allin;


			return true;
		}

		public Text terminate() {
			if(result == null) {
				return new Text("0t0t0t0");
			}

			String s=""+result.total+"t"+result.win+"t"+result.fold+"t"+result.allin;
			return new Text(s);
		}
	}
}

如非注明转载, 均为原创. 本站遵循知识共享CC协议,转载请注明来源

FEATURED TAGS
css vc6 http automake linux make makefile voip 乱码 awk flash vista vi vim javascript pietty putty ssh posix subversion svn windows 删除 编译 多线程 wxwidgets ie ubuntu 开源 c python bash 备份 性能 scp 汉字 log ruby 中文 bug msn nginx php shell wordpress mqueue android eclipse java mac ios html5 js mysql protobuf apache hadoop install iocp twisted centos mapreduce hbase thrift tutorial hive erlang lucene hdfs sqoop utf8 filter 草原 yarn ganglia 恢复 scrapy django fsimage flume tail flume-ng mining scala go kafka gradle cassandra baas spring postgres maven mybatis mongodb https nodejs 镜像 心理学 机器学习 Keras theano anaconda docker spark akka-http json 群论 区块链 加密 抽象代数 离散对数 同余 欧拉函数 扩展欧几里德算法 ES6 node-inspect debug win10 vscode 挖矿
FRIENDS