开源 hbase的example/mapreduce里有个类IndexBuilder是用来对已有表建索引的。其代码有一点点需要修改
conf.set(TableInputFormat.SCAN, TableMapReduceUtil.convertScanToString(new Scan()));conf.set(TableInputFormat.SCAN, convertScanToString(new Scan()));
上面的改为下面的，为什么呢，因为TableMapReduceUtil类的那个函数是private的（也许以前是public吧，我用的是hbase-0.90.3），所以需要将那个函数的代码复制到当前任务类里
好了，上代码吧：
/** * Copyright 2009 The Apache Software Foundation * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */package hbase.test;import java.io.ByteArrayOutputStream;import java.io.DataOutputStream;import java.io.IOException;import java.util.HashMap;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.hbase.HBaseConfiguration;import org.apache.hadoop.hbase.client.Put;import org.apache.hadoop.hbase.client.Result;import org.apache.hadoop.hbase.client.Scan;import org.apache.hadoop.hbase.io.ImmutableBytesWritable;import org.apache.hadoop.hbase.mapreduce.MultiTableOutputFormat;import org.apache.hadoop.hbase.mapreduce.TableInputFormat;import org.apache.hadoop.hbase.util.Base64;import org.apache.hadoop.hbase.util.Bytes;import org.apache.hadoop.io.Writable;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.util.GenericOptionsParser;/** * Example map/reduce job to construct index tables that can be used to quickly * find a row based on the value of a column. It demonstrates: * <ul> * <li>Using TableInputFormat and TableMapReduceUtil to use an HTable as input * to a map/reduce job.</li> * <li>Passing values from main method to children via the configuration.</li> * <li>Using MultiTableOutputFormat to output to multiple tables from a * map/reduce job.</li> * <li>A real use case of building a secondary index over a table.</li> * </ul> * * <h3>Usage</h3> * * <p> * Modify ${HADOOP_HOME}/conf/hadoop-env.sh to include the hbase jar, the * zookeeper jar, the examples output directory, and the hbase conf directory in * HADOOP_CLASSPATH, and then run * <tt><strong>bin/hadoop org.apache.hadoop.hbase.mapreduce.IndexBuilder TABLE_NAME COLUMN_FAMILY ATTR [ATTR …]</strong></tt> * </p> * * <p> * To run with the sample data provided in index-builder-setup.rb, use the * arguments <strong><tt>people attributes name email phone</tt></strong>. * </p> * * <p> * This code was written against HBase 0.21 trunk. * </p> */public class IndexBuilder { /** the column family containing the indexed row key */ public static final byte[] INDEX_COLUMN = Bytes.toBytes("INDEX"); /** the qualifier containing the indexed row key */ public static final byte[] INDEX_QUALIFIER = Bytes.toBytes("ROW");/*** Internal Mapper to be run by Hadoop.*/ public static class Map extendsMapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Writable> {private byte[] family;private HashMap<byte[], ImmutableBytesWritable> indexes;@Overrideprotected void map(ImmutableBytesWritable rowKey, Result result, Context context)throws IOException, InterruptedException {for(java.util.Map.Entry<byte[], ImmutableBytesWritable> index : indexes.entrySet()) {byte[] qualifier = index.getKey();ImmutableBytesWritable tableName = index.getValue();byte[] value = result.getValue(family, qualifier);if (value != null) {// original: row 123 attribute:phone 555-1212// index: row 555-1212 INDEX:ROW 123Put put = new Put(value);put.add(INDEX_COLUMN, INDEX_QUALIFIER, rowKey.get());context.write(tableName, put);}}}@Overrideprotected void setup(Context context) throws IOException,InterruptedException {Configuration configuration = context.getConfiguration();String tableName = configuration.get("index.tablename");String[] fields = configuration.getStrings("index.fields");String familyName = configuration.get("index.familyname");family = Bytes.toBytes(familyName);indexes = new HashMap<byte[], ImmutableBytesWritable>();for(String field : fields) {// if the table is "people" and the field to index is "email", then the// index table will be called "people-email"indexes.put(Bytes.toBytes(field),new ImmutableBytesWritable(Bytes.toBytes(tableName + "-" + field)));}} }/*** Job configuration.*/ public static Job configureJob(Configuration conf, String [] args) throws IOException {String tableName = args[0];String columnFamily = args[1];System.out.println("****" + tableName);conf.set(TableInputFormat.SCAN, convertScanToString(new Scan()));conf.set(TableInputFormat.INPUT_TABLE, tableName);conf.set("index.tablename", tableName);conf.set("index.familyname", columnFamily);String[] fields = new String[args.length – 2];for(int i = 0; i < fields.length; i++) {fields[i] = args[i + 2];}conf.setStrings("index.fields", fields);//conf.set("index.familyname", "attributes");Job job = new Job(conf, tableName);job.setJarByClass(IndexBuilder.class);job.setMapperClass(Map.class);job.setNumReduceTasks(0);job.setInputFormatClass(TableInputFormat.class);job.setOutputFormatClass(MultiTableOutputFormat.class);return job; }public static void main(String[] args) throws Exception {Configuration conf = HBaseConfiguration.create();conf.set("hbase.zookeeper.quorum", "node2,node4,node3");//conf.set("fs.default.name","hdfs://node1");//conf.set("mapred.job.tracker","node1:54311");String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();if(otherArgs.length < 3) {System.err.println("Only " + otherArgs.length + " arguments supplied, required: 3");System.err.println("Usage: IndexBuilder <TABLE_NAME> <COLUMN_FAMILY> <ATTR> [<ATTR> …]");System.exit(-1);}Job job = configureJob(conf, otherArgs);System.exit(job.waitForCompletion(true) ? 0 : 1); }private static String convertScanToString(Scan scan) throws IOException {ByteArrayOutputStream out = new ByteArrayOutputStream();DataOutputStream dos = new DataOutputStream(out);scan.write(dos);return Base64.encodeBytes(out.toByteArray());}}
大家看到了，我是放在hbase.test包里的，于是我将这个包打成jar包，然后放到hadoop集群里，执行hadoop jar mapreduceindexbuilder.jar ‘it’, ‘f1’, ‘q1’
之所以这样写，因为我export的时候已经选了类了，所以不用再输类名，接下来的三个参数分别是 表名，列族名，修饰符名
只是简单的测试而已。
一直报错。
风景如何，其实并不重要。重要的是，你在我的身边。
