View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.mapreduce;
19  
20  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
21  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
22  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
23  
24  import java.io.ByteArrayOutputStream;
25  import java.io.DataOutputStream;
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.Collections;
29  import java.util.HashMap;
30  import java.util.List;
31  import java.util.Map;
32  
33  import org.apache.hadoop.classification.InterfaceAudience;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.KeyValue.Type;
37  import org.apache.hadoop.hbase.Tag;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.client.Result;
40  import org.apache.hadoop.hbase.client.ResultScanner;
41  import org.apache.hadoop.hbase.client.Scan;
42  import org.apache.hadoop.hbase.io.util.StreamUtils;
43  import org.apache.hadoop.hbase.mapreduce.ImportTsv.TsvParser.BadTsvLineException;
44  import org.apache.hadoop.hbase.security.visibility.Authorizations;
45  import org.apache.hadoop.hbase.security.visibility.ExpressionExpander;
46  import org.apache.hadoop.hbase.security.visibility.ExpressionParser;
47  import org.apache.hadoop.hbase.security.visibility.InvalidLabelException;
48  import org.apache.hadoop.hbase.security.visibility.ParseException;
49  import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
50  import org.apache.hadoop.hbase.security.visibility.expression.ExpressionNode;
51  import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode;
52  import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode;
53  import org.apache.hadoop.hbase.security.visibility.expression.Operator;
54  import org.apache.hadoop.hbase.util.Bytes;
55  
56  /**
57   * An utility class that helps the mapper and reducers used with visibility to
58   * scan the visibility_labels and helps in parsing and expanding the visibility
59   * tags
60   * 
61   */
62  @InterfaceAudience.Private
63  public class LabelExpander {
64    private Configuration conf;
65    private ExpressionParser parser = new ExpressionParser();
66    private ExpressionExpander expander = new ExpressionExpander();
67  
68    public LabelExpander(Configuration conf) {
69      this.conf = conf;
70    }
71  
72    private Map<String, Integer> labels;
73  
74    // TODO : The code repeats from that in Visibility Controller.. Refactoring
75    // may be needed
76    private List<Tag> createVisibilityTags(String visibilityLabelsExp) throws IOException,
77        ParseException, InvalidLabelException {
78      ExpressionNode node = null;
79      node = parser.parse(visibilityLabelsExp);
80      node = expander.expand(node);
81      List<Tag> tags = new ArrayList<Tag>();
82      ByteArrayOutputStream baos = new ByteArrayOutputStream();
83      DataOutputStream dos = new DataOutputStream(baos);
84      List<Integer> labelOrdinals = new ArrayList<Integer>();
85      // We will be adding this tag before the visibility tags and the presence of
86      // this
87      // tag indicates we are supporting deletes with cell visibility
88      tags.add(VisibilityUtils.VIS_SERIALIZATION_TAG);
89      if (node.isSingleNode()) {
90        getLabelOrdinals(node, labelOrdinals);
91        writeLabelOrdinalsToStream(labelOrdinals, dos);
92        tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
93        baos.reset();
94      } else {
95        NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node;
96        if (nlNode.getOperator() == Operator.OR) {
97          for (ExpressionNode child : nlNode.getChildExps()) {
98            getLabelOrdinals(child, labelOrdinals);
99            writeLabelOrdinalsToStream(labelOrdinals, dos);
100           tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
101           baos.reset();
102           labelOrdinals.clear();
103         }
104       } else {
105         getLabelOrdinals(nlNode, labelOrdinals);
106         writeLabelOrdinalsToStream(labelOrdinals, dos);
107         tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
108         baos.reset();
109       }
110     }
111     return tags;
112   }
113 
114   private void writeLabelOrdinalsToStream(List<Integer> labelOrdinals, DataOutputStream dos)
115       throws IOException {
116     Collections.sort(labelOrdinals);
117     for (Integer labelOrdinal : labelOrdinals) {
118       StreamUtils.writeRawVInt32(dos, labelOrdinal);
119     }
120   }
121 
122   private void getLabelOrdinals(ExpressionNode node, List<Integer> labelOrdinals)
123       throws IOException, InvalidLabelException {
124     if (node.isSingleNode()) {
125       String identifier = null;
126       int labelOrdinal = 0;
127       if (node instanceof LeafExpressionNode) {
128         identifier = ((LeafExpressionNode) node).getIdentifier();
129         labelOrdinal = this.labels.get(identifier);
130       } else {
131         // This is a NOT node.
132         LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node)
133             .getChildExps().get(0);
134         identifier = lNode.getIdentifier();
135         labelOrdinal = this.labels.get(identifier);
136         labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal.
137       }
138       if (labelOrdinal == 0) {
139         throw new InvalidLabelException("Invalid visibility label " + identifier);
140       }
141       labelOrdinals.add(labelOrdinal);
142     } else {
143       List<ExpressionNode> childExps = ((NonLeafExpressionNode) node).getChildExps();
144       for (ExpressionNode child : childExps) {
145         getLabelOrdinals(child, labelOrdinals);
146       }
147     }
148   }
149 
150   private void createLabels() throws IOException {
151     // This scan should be done by user with global_admin previliges.. Ensure
152     // that it works
153     HTable visibilityLabelsTable = null;
154     ResultScanner scanner = null;
155     try {
156       labels = new HashMap<String, Integer>();
157       visibilityLabelsTable = new HTable(conf, LABELS_TABLE_NAME.getName());
158       Scan scan = new Scan();
159       scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));
160       scan.addColumn(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
161       scanner = visibilityLabelsTable.getScanner(scan);
162       while (true) {
163         Result next = scanner.next();
164         if (next == null) {
165           break;
166         }
167         byte[] row = next.getRow();
168         byte[] value = next.getValue(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
169         labels.put(Bytes.toString(value), Bytes.toInt(row));
170       }
171     } finally {
172       try {
173         if (scanner != null) {
174           scanner.close();
175         }
176       } finally {
177         if (visibilityLabelsTable != null) {
178           visibilityLabelsTable.close();
179         }
180       }
181     }
182   }
183 
184   /**
185    * Creates a kv from the cell visibility expr specified in the ImportTSV and uses it as the
186    * visibility tag in the kv
187    * @param rowKeyOffset
188    * @param rowKeyLength
189    * @param family
190    * @param familyOffset
191    * @param familyLength
192    * @param qualifier
193    * @param qualifierOffset
194    * @param qualifierLength
195    * @param ts
196    * @param put
197    * @param lineBytes
198    * @param columnOffset
199    * @param columnLength
200    * @param cellVisibilityExpr
201    * @return KeyValue from the cell visibility expr
202    * @throws IOException
203    * @throws BadTsvLineException
204    * @throws ParseException 
205    */
206   public KeyValue createKVFromCellVisibilityExpr(int rowKeyOffset, int rowKeyLength, byte[] family,
207       int familyOffset, int familyLength, byte[] qualifier, int qualifierOffset,
208       int qualifierLength, long ts, Type put, byte[] lineBytes, int columnOffset, int columnLength,
209       String cellVisibilityExpr) throws IOException, BadTsvLineException {
210     if(this.labels == null  && cellVisibilityExpr != null) {
211       createLabels();
212     }
213     KeyValue kv = null;
214     if (cellVisibilityExpr != null) {
215       // Apply the expansion and parsing here
216       try {
217         List<Tag> visibilityTags = createVisibilityTags(cellVisibilityExpr);
218         kv = new KeyValue(lineBytes, rowKeyOffset, rowKeyLength, family, familyOffset,
219             familyLength, qualifier, qualifierOffset, qualifierLength, ts, KeyValue.Type.Put,
220             lineBytes, columnOffset, columnLength, visibilityTags);
221       } catch (ParseException e) {
222         throw new BadTsvLineException("Parse Exception " + e.getMessage());
223       }
224     } else {
225       kv = new KeyValue(lineBytes, rowKeyOffset, rowKeyLength, family, familyOffset, familyLength,
226           qualifier, qualifierOffset, qualifierLength, ts, KeyValue.Type.Put, lineBytes, columnOffset,
227           columnLength);
228     }
229     return kv;
230   }
231 }