1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.FileNotFoundException;
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.UUID;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.classification.InterfaceAudience;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FSDataInputStream;
35 import org.apache.hadoop.fs.FSDataOutputStream;
36 import org.apache.hadoop.fs.FileStatus;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.FileUtil;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.fs.PathFilter;
41 import org.apache.hadoop.fs.permission.FsPermission;
42 import org.apache.hadoop.hbase.HColumnDescriptor;
43 import org.apache.hadoop.hbase.HConstants;
44 import org.apache.hadoop.hbase.HRegionInfo;
45 import org.apache.hadoop.hbase.HTableDescriptor;
46 import org.apache.hadoop.hbase.KeyValue;
47 import org.apache.hadoop.hbase.backup.HFileArchiver;
48 import org.apache.hadoop.hbase.fs.HFileSystem;
49 import org.apache.hadoop.hbase.io.Reference;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.FSHDFSUtils;
52 import org.apache.hadoop.hbase.util.FSUtils;
53 import org.apache.hadoop.hbase.util.Threads;
54
55
56
57
58
59 @InterfaceAudience.Private
60 public class HRegionFileSystem {
61 public static final Log LOG = LogFactory.getLog(HRegionFileSystem.class);
62
63
64 public final static String REGION_INFO_FILE = ".regioninfo";
65
66
67 public static final String REGION_MERGES_DIR = ".merges";
68
69
70 public static final String REGION_SPLITS_DIR = ".splits";
71
72
73 private static final String REGION_TEMP_DIR = ".tmp";
74
75 private final HRegionInfo regionInfo;
76 private final Configuration conf;
77 private final Path tableDir;
78 private final FileSystem fs;
79
80
81
82
83
84 private final int hdfsClientRetriesNumber;
85 private final int baseSleepBeforeRetries;
86 private static final int DEFAULT_HDFS_CLIENT_RETRIES_NUMBER = 10;
87 private static final int DEFAULT_BASE_SLEEP_BEFORE_RETRIES = 1000;
88
89
90
91
92
93
94
95
96 HRegionFileSystem(final Configuration conf, final FileSystem fs, final Path tableDir,
97 final HRegionInfo regionInfo) {
98 this.fs = fs;
99 this.conf = conf;
100 this.tableDir = tableDir;
101 this.regionInfo = regionInfo;
102 this.hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
103 DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
104 this.baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
105 DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
106 }
107
108
109 public FileSystem getFileSystem() {
110 return this.fs;
111 }
112
113
114 public HRegionInfo getRegionInfo() {
115 return this.regionInfo;
116 }
117
118
119 public Path getTableDir() {
120 return this.tableDir;
121 }
122
123
124 public Path getRegionDir() {
125 return new Path(this.tableDir, this.regionInfo.getEncodedName());
126 }
127
128
129
130
131
132 Path getTempDir() {
133 return new Path(getRegionDir(), REGION_TEMP_DIR);
134 }
135
136
137
138
139 void cleanupTempDir() throws IOException {
140 deleteDir(getTempDir());
141 }
142
143
144
145
146
147
148
149
150
151 Path getStoreDir(final String familyName) {
152 return new Path(this.getRegionDir(), familyName);
153 }
154
155
156
157
158
159
160
161 Path createStoreDir(final String familyName) throws IOException {
162 Path storeDir = getStoreDir(familyName);
163 if(!fs.exists(storeDir) && !createDir(storeDir))
164 throw new IOException("Failed creating "+storeDir);
165 return storeDir;
166 }
167
168
169
170
171
172
173
174 public Collection<StoreFileInfo> getStoreFiles(final byte[] familyName) throws IOException {
175 return getStoreFiles(Bytes.toString(familyName));
176 }
177
178
179
180
181
182
183
184 public Collection<StoreFileInfo> getStoreFiles(final String familyName) throws IOException {
185 Path familyDir = getStoreDir(familyName);
186 FileStatus[] files = FSUtils.listStatus(this.fs, familyDir);
187 if (files == null) return null;
188
189 ArrayList<StoreFileInfo> storeFiles = new ArrayList<StoreFileInfo>(files.length);
190 for (FileStatus status: files) {
191 if (!StoreFileInfo.isValid(status)) continue;
192
193 storeFiles.add(new StoreFileInfo(this.conf, this.fs, status));
194 }
195 return storeFiles;
196 }
197
198
199
200
201
202
203
204
205 Path getStoreFilePath(final String familyName, final String fileName) {
206 Path familyDir = getStoreDir(familyName);
207 return new Path(familyDir, fileName).makeQualified(this.fs);
208 }
209
210
211
212
213
214
215
216
217 StoreFileInfo getStoreFileInfo(final String familyName, final String fileName)
218 throws IOException {
219 Path familyDir = getStoreDir(familyName);
220 FileStatus status = fs.getFileStatus(new Path(familyDir, fileName));
221 return new StoreFileInfo(this.conf, this.fs, status);
222 }
223
224
225
226
227
228
229
230 public boolean hasReferences(final String familyName) throws IOException {
231 FileStatus[] files = FSUtils.listStatus(fs, getStoreDir(familyName),
232 new PathFilter () {
233 public boolean accept(Path path) {
234 return StoreFileInfo.isReference(path);
235 }
236 }
237 );
238 return files != null && files.length > 0;
239 }
240
241
242
243
244
245
246
247 public boolean hasReferences(final HTableDescriptor htd) throws IOException {
248 for (HColumnDescriptor family : htd.getFamilies()) {
249 if (hasReferences(family.getNameAsString())) {
250 return true;
251 }
252 }
253 return false;
254 }
255
256
257
258
259
260 public Collection<String> getFamilies() throws IOException {
261 FileStatus[] fds = FSUtils.listStatus(fs, getRegionDir(), new FSUtils.FamilyDirFilter(fs));
262 if (fds == null) return null;
263
264 ArrayList<String> families = new ArrayList<String>(fds.length);
265 for (FileStatus status: fds) {
266 families.add(status.getPath().getName());
267 }
268
269 return families;
270 }
271
272
273
274
275
276
277 public void deleteFamily(final String familyName) throws IOException {
278
279 HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, Bytes.toBytes(familyName));
280
281
282 Path familyDir = getStoreDir(familyName);
283 if(fs.exists(familyDir) && !deleteDir(familyDir))
284 throw new IOException("Could not delete family " + familyName
285 + " from FileSystem for region " + regionInfo.getRegionNameAsString() + "("
286 + regionInfo.getEncodedName() + ")");
287 }
288
289
290
291
292
293
294 private static String generateUniqueName(final String suffix) {
295 String name = UUID.randomUUID().toString().replaceAll("-", "");
296 if (suffix != null) name += suffix;
297 return name;
298 }
299
300
301
302
303
304
305
306
307
308
309
310
311 public Path createTempName() {
312 return createTempName(null);
313 }
314
315
316
317
318
319
320
321
322
323
324
325
326
327 public Path createTempName(final String suffix) {
328 return new Path(getTempDir(), generateUniqueName(suffix));
329 }
330
331
332
333
334
335
336
337
338 public Path commitStoreFile(final String familyName, final Path buildPath) throws IOException {
339 return commitStoreFile(familyName, buildPath, -1, false);
340 }
341
342
343
344
345
346
347
348
349
350
351 private Path commitStoreFile(final String familyName, final Path buildPath,
352 final long seqNum, final boolean generateNewName) throws IOException {
353 Path storeDir = getStoreDir(familyName);
354 if(!fs.exists(storeDir) && !createDir(storeDir))
355 throw new IOException("Failed creating " + storeDir);
356
357 String name = buildPath.getName();
358 if (generateNewName) {
359 name = generateUniqueName((seqNum < 0) ? null : "_SeqId_" + seqNum + "_");
360 }
361 Path dstPath = new Path(storeDir, name);
362 if (!fs.exists(buildPath)) {
363 throw new FileNotFoundException(buildPath.toString());
364 }
365 LOG.debug("Committing store file " + buildPath + " as " + dstPath);
366
367 if (!rename(buildPath, dstPath)) {
368 throw new IOException("Failed rename of " + buildPath + " to " + dstPath);
369 }
370 return dstPath;
371 }
372
373
374
375
376
377
378
379 void commitStoreFiles(final Map<byte[], List<StoreFile>> storeFiles) throws IOException {
380 for (Map.Entry<byte[], List<StoreFile>> es: storeFiles.entrySet()) {
381 String familyName = Bytes.toString(es.getKey());
382 for (StoreFile sf: es.getValue()) {
383 commitStoreFile(familyName, sf.getPath());
384 }
385 }
386 }
387
388
389
390
391
392
393
394 public void removeStoreFile(final String familyName, final Path filePath)
395 throws IOException {
396 HFileArchiver.archiveStoreFile(this.conf, this.fs, this.regionInfo,
397 this.tableDir, Bytes.toBytes(familyName), filePath);
398 }
399
400
401
402
403
404
405
406 public void removeStoreFiles(final String familyName, final Collection<StoreFile> storeFiles)
407 throws IOException {
408 HFileArchiver.archiveStoreFiles(this.conf, this.fs, this.regionInfo,
409 this.tableDir, Bytes.toBytes(familyName), storeFiles);
410 }
411
412
413
414
415
416
417
418
419
420
421
422
423 Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
424 throws IOException {
425
426 FileSystem srcFs = srcPath.getFileSystem(conf);
427 FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
428
429
430
431
432 if (!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)) {
433 LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " +
434 "the destination store. Copying file over to destination filesystem.");
435 Path tmpPath = createTempName();
436 FileUtil.copy(srcFs, srcPath, fs, tmpPath, false, conf);
437 LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath);
438 srcPath = tmpPath;
439 }
440
441 return commitStoreFile(familyName, srcPath, seqNum, true);
442 }
443
444
445
446
447
448 Path getSplitsDir() {
449 return new Path(getRegionDir(), REGION_SPLITS_DIR);
450 }
451
452 Path getSplitsDir(final HRegionInfo hri) {
453 return new Path(getSplitsDir(), hri.getEncodedName());
454 }
455
456
457
458
459 void cleanupSplitsDir() throws IOException {
460 deleteDir(getSplitsDir());
461 }
462
463
464
465
466
467
468
469 void cleanupAnySplitDetritus() throws IOException {
470 Path splitdir = this.getSplitsDir();
471 if (!fs.exists(splitdir)) return;
472
473
474
475
476
477
478
479 FileStatus[] daughters = FSUtils.listStatus(fs, splitdir, new FSUtils.DirFilter(fs));
480 if (daughters != null) {
481 for (FileStatus daughter: daughters) {
482 Path daughterDir = new Path(getTableDir(), daughter.getPath().getName());
483 if (fs.exists(daughterDir) && !deleteDir(daughterDir)) {
484 throw new IOException("Failed delete of " + daughterDir);
485 }
486 }
487 }
488 cleanupSplitsDir();
489 LOG.info("Cleaned up old failed split transaction detritus: " + splitdir);
490 }
491
492
493
494
495
496
497 void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException {
498 Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
499 if (this.fs.exists(regionDir) && !deleteDir(regionDir)) {
500 throw new IOException("Failed delete of " + regionDir);
501 }
502 }
503
504
505
506
507
508
509
510 Path commitDaughterRegion(final HRegionInfo regionInfo) throws IOException {
511 Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName());
512 Path daughterTmpDir = this.getSplitsDir(regionInfo);
513 if (fs.exists(daughterTmpDir)) {
514
515 Path regionInfoFile = new Path(daughterTmpDir, REGION_INFO_FILE);
516 byte[] regionInfoContent = getRegionInfoFileContent(regionInfo);
517 writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
518
519
520 if (!rename(daughterTmpDir, regionDir)) {
521 throw new IOException("Unable to rename " + daughterTmpDir + " to " + regionDir);
522 }
523 }
524 return regionDir;
525 }
526
527
528
529
530 void createSplitsDir() throws IOException {
531 Path splitdir = getSplitsDir();
532 if (fs.exists(splitdir)) {
533 LOG.info("The " + splitdir + " directory exists. Hence deleting it to recreate it");
534 if (!deleteDir(splitdir)) {
535 throw new IOException("Failed deletion of " + splitdir
536 + " before creating them again.");
537 }
538 }
539
540 if (!createDir(splitdir)) {
541 throw new IOException("Failed create of " + splitdir);
542 }
543 }
544
545
546
547
548
549
550
551
552
553
554
555
556 Path splitStoreFile(final HRegionInfo hri, final String familyName,
557 final StoreFile f, final byte[] splitRow, final boolean top) throws IOException {
558
559
560
561 if (top) {
562
563 KeyValue splitKey = KeyValue.createFirstOnRow(splitRow);
564 byte[] lastKey = f.createReader().getLastKey();
565
566 if (lastKey == null) return null;
567 if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(),
568 splitKey.getKeyOffset(), splitKey.getKeyLength(), lastKey, 0, lastKey.length) > 0) {
569 return null;
570 }
571 } else {
572
573 KeyValue splitKey = KeyValue.createLastOnRow(splitRow);
574 byte[] firstKey = f.createReader().getFirstKey();
575
576 if (firstKey == null) return null;
577 if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(),
578 splitKey.getKeyOffset(), splitKey.getKeyLength(), firstKey, 0, firstKey.length) < 0) {
579 return null;
580 }
581 }
582
583 f.getReader().close(true);
584
585 Path splitDir = new Path(getSplitsDir(hri), familyName);
586
587 Reference r =
588 top ? Reference.createTopReference(splitRow): Reference.createBottomReference(splitRow);
589
590
591
592
593 String parentRegionName = regionInfo.getEncodedName();
594
595
596 Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName);
597 return r.write(fs, p);
598 }
599
600
601
602
603
604 Path getMergesDir() {
605 return new Path(getRegionDir(), REGION_MERGES_DIR);
606 }
607
608 Path getMergesDir(final HRegionInfo hri) {
609 return new Path(getMergesDir(), hri.getEncodedName());
610 }
611
612
613
614
615 void cleanupMergesDir() throws IOException {
616 deleteDir(getMergesDir());
617 }
618
619
620
621
622
623
624 void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException {
625 Path regionDir = new Path(this.tableDir, mergedRegion.getEncodedName());
626 if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) {
627 throw new IOException("Failed delete of " + regionDir);
628 }
629 }
630
631
632
633
634
635
636 void createMergesDir() throws IOException {
637 Path mergesdir = getMergesDir();
638 if (fs.exists(mergesdir)) {
639 LOG.info("The " + mergesdir
640 + " directory exists. Hence deleting it to recreate it");
641 if (!fs.delete(mergesdir, true)) {
642 throw new IOException("Failed deletion of " + mergesdir
643 + " before creating them again.");
644 }
645 }
646 if (!fs.mkdirs(mergesdir))
647 throw new IOException("Failed create of " + mergesdir);
648 }
649
650
651
652
653
654
655
656
657
658
659
660 Path mergeStoreFile(final HRegionInfo mergedRegion, final String familyName,
661 final StoreFile f, final Path mergedDir)
662 throws IOException {
663 Path referenceDir = new Path(new Path(mergedDir,
664 mergedRegion.getEncodedName()), familyName);
665
666 Reference r = Reference.createTopReference(regionInfo.getStartKey());
667
668
669
670
671 String mergingRegionName = regionInfo.getEncodedName();
672
673
674 Path p = new Path(referenceDir, f.getPath().getName() + "."
675 + mergingRegionName);
676 return r.write(fs, p);
677 }
678
679
680
681
682
683
684
685 void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException {
686 Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName());
687 Path mergedRegionTmpDir = this.getMergesDir(mergedRegionInfo);
688
689 if (mergedRegionTmpDir != null && fs.exists(mergedRegionTmpDir)) {
690 if (!fs.rename(mergedRegionTmpDir, regionDir)) {
691 throw new IOException("Unable to rename " + mergedRegionTmpDir + " to "
692 + regionDir);
693 }
694 }
695 }
696
697
698
699
700
701
702
703
704
705 void logFileSystemState(final Log LOG) throws IOException {
706 FSUtils.logFileSystemState(fs, this.getRegionDir(), LOG);
707 }
708
709
710
711
712
713
714 private static byte[] getRegionInfoFileContent(final HRegionInfo hri) throws IOException {
715 return hri.toDelimitedByteArray();
716 }
717
718
719
720
721
722
723
724
725 public static HRegionInfo loadRegionInfoFileContent(final FileSystem fs, final Path regionDir)
726 throws IOException {
727 FSDataInputStream in = fs.open(new Path(regionDir, REGION_INFO_FILE));
728 try {
729 return HRegionInfo.parseFrom(in);
730 } finally {
731 in.close();
732 }
733 }
734
735
736
737
738 private static void writeRegionInfoFileContent(final Configuration conf, final FileSystem fs,
739 final Path regionInfoFile, final byte[] content) throws IOException {
740
741 FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
742
743 FSDataOutputStream out = FSUtils.create(fs, regionInfoFile, perms, null);
744 try {
745 out.write(content);
746 } finally {
747 out.close();
748 }
749 }
750
751
752
753
754
755 void checkRegionInfoOnFilesystem() throws IOException {
756
757
758
759
760
761 byte[] content = getRegionInfoFileContent(regionInfo);
762 try {
763 Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
764
765 FileStatus status = fs.getFileStatus(regionInfoFile);
766 if (status != null && status.getLen() == content.length) {
767
768
769 return;
770 }
771
772 LOG.info("Rewriting .regioninfo file at: " + regionInfoFile);
773 if (!fs.delete(regionInfoFile, false)) {
774 throw new IOException("Unable to remove existing " + regionInfoFile);
775 }
776 } catch (FileNotFoundException e) {
777 LOG.warn(REGION_INFO_FILE + " file not found for region: " + regionInfo.getEncodedName());
778 }
779
780
781 writeRegionInfoOnFilesystem(content, true);
782 }
783
784
785
786
787
788 private void writeRegionInfoOnFilesystem(boolean useTempDir) throws IOException {
789 byte[] content = getRegionInfoFileContent(regionInfo);
790 writeRegionInfoOnFilesystem(content, useTempDir);
791 }
792
793
794
795
796
797
798 private void writeRegionInfoOnFilesystem(final byte[] regionInfoContent,
799 final boolean useTempDir) throws IOException {
800 Path regionInfoFile = new Path(getRegionDir(), REGION_INFO_FILE);
801 if (useTempDir) {
802
803
804
805
806
807
808 Path tmpPath = new Path(getTempDir(), REGION_INFO_FILE);
809
810
811
812
813
814 if (FSUtils.isExists(fs, tmpPath)) {
815 FSUtils.delete(fs, tmpPath, true);
816 }
817
818
819 writeRegionInfoFileContent(conf, fs, tmpPath, regionInfoContent);
820
821
822 if (fs.exists(tmpPath) && !rename(tmpPath, regionInfoFile)) {
823 throw new IOException("Unable to rename " + tmpPath + " to " + regionInfoFile);
824 }
825 } else {
826
827 writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent);
828 }
829 }
830
831
832
833
834
835
836
837
838
839 public static HRegionFileSystem createRegionOnFileSystem(final Configuration conf,
840 final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
841 HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
842 Path regionDir = regionFs.getRegionDir();
843
844 if (fs.exists(regionDir)) {
845 LOG.warn("Trying to create a region that already exists on disk: " + regionDir);
846 throw new IOException("The specified region already exists on disk: " + regionDir);
847 }
848
849
850 if (!createDirOnFileSystem(fs, conf, regionDir)) {
851 LOG.warn("Unable to create the region directory: " + regionDir);
852 throw new IOException("Unable to create region directory: " + regionDir);
853 }
854
855
856 regionFs.writeRegionInfoOnFilesystem(false);
857 return regionFs;
858 }
859
860
861
862
863
864
865
866
867
868
869 public static HRegionFileSystem openRegionFromFileSystem(final Configuration conf,
870 final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo, boolean readOnly)
871 throws IOException {
872 HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
873 Path regionDir = regionFs.getRegionDir();
874
875 if (!fs.exists(regionDir)) {
876 LOG.warn("Trying to open a region that do not exists on disk: " + regionDir);
877 throw new IOException("The specified region do not exists on disk: " + regionDir);
878 }
879
880 if (!readOnly) {
881
882 regionFs.cleanupTempDir();
883 regionFs.cleanupSplitsDir();
884 regionFs.cleanupMergesDir();
885
886
887 regionFs.checkRegionInfoOnFilesystem();
888 }
889
890 return regionFs;
891 }
892
893
894
895
896
897
898
899
900
901 public static void deleteRegionFromFileSystem(final Configuration conf,
902 final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException {
903 HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo);
904 Path regionDir = regionFs.getRegionDir();
905
906 if (!fs.exists(regionDir)) {
907 LOG.warn("Trying to delete a region that do not exists on disk: " + regionDir);
908 return;
909 }
910
911 if (LOG.isDebugEnabled()) {
912 LOG.debug("DELETING region " + regionDir);
913 }
914
915
916 Path rootDir = FSUtils.getRootDir(conf);
917 HFileArchiver.archiveRegion(fs, rootDir, tableDir, regionDir);
918
919
920 if (!fs.delete(regionDir, true)) {
921 LOG.warn("Failed delete of " + regionDir);
922 }
923 }
924
925
926
927
928
929
930
931
932 boolean createDir(Path dir) throws IOException {
933 int i = 0;
934 IOException lastIOE = null;
935 do {
936 try {
937 return fs.mkdirs(dir);
938 } catch (IOException ioe) {
939 lastIOE = ioe;
940 if (fs.exists(dir)) return true;
941 sleepBeforeRetry("Create Directory", i+1);
942 }
943 } while (++i <= hdfsClientRetriesNumber);
944 throw new IOException("Exception in createDir", lastIOE);
945 }
946
947
948
949
950
951
952
953
954 boolean rename(Path srcpath, Path dstPath) throws IOException {
955 IOException lastIOE = null;
956 int i = 0;
957 do {
958 try {
959 return fs.rename(srcpath, dstPath);
960 } catch (IOException ioe) {
961 lastIOE = ioe;
962 if (!fs.exists(srcpath) && fs.exists(dstPath)) return true;
963
964 sleepBeforeRetry("Rename Directory", i+1);
965 }
966 } while (++i <= hdfsClientRetriesNumber);
967 throw new IOException("Exception in rename", lastIOE);
968 }
969
970
971
972
973
974
975
976 boolean deleteDir(Path dir) throws IOException {
977 IOException lastIOE = null;
978 int i = 0;
979 do {
980 try {
981 return fs.delete(dir, true);
982 } catch (IOException ioe) {
983 lastIOE = ioe;
984 if (!fs.exists(dir)) return true;
985
986 sleepBeforeRetry("Delete Directory", i+1);
987 }
988 } while (++i <= hdfsClientRetriesNumber);
989 throw new IOException("Exception in DeleteDir", lastIOE);
990 }
991
992
993
994
995 private void sleepBeforeRetry(String msg, int sleepMultiplier) {
996 sleepBeforeRetry(msg, sleepMultiplier, baseSleepBeforeRetries, hdfsClientRetriesNumber);
997 }
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009 private static boolean createDirOnFileSystem(FileSystem fs, Configuration conf, Path dir)
1010 throws IOException {
1011 int i = 0;
1012 IOException lastIOE = null;
1013 int hdfsClientRetriesNumber = conf.getInt("hdfs.client.retries.number",
1014 DEFAULT_HDFS_CLIENT_RETRIES_NUMBER);
1015 int baseSleepBeforeRetries = conf.getInt("hdfs.client.sleep.before.retries",
1016 DEFAULT_BASE_SLEEP_BEFORE_RETRIES);
1017 do {
1018 try {
1019 return fs.mkdirs(dir);
1020 } catch (IOException ioe) {
1021 lastIOE = ioe;
1022 if (fs.exists(dir)) return true;
1023 sleepBeforeRetry("Create Directory", i+1, baseSleepBeforeRetries, hdfsClientRetriesNumber);
1024 }
1025 } while (++i <= hdfsClientRetriesNumber);
1026 throw new IOException("Exception in createDir", lastIOE);
1027 }
1028
1029
1030
1031
1032
1033 private static void sleepBeforeRetry(String msg, int sleepMultiplier, int baseSleepBeforeRetries,
1034 int hdfsClientRetriesNumber) {
1035 if (sleepMultiplier > hdfsClientRetriesNumber) {
1036 LOG.debug(msg + ", retries exhausted");
1037 return;
1038 }
1039 LOG.debug(msg + ", sleeping " + baseSleepBeforeRetries + " times " + sleepMultiplier);
1040 Threads.sleep((long)baseSleepBeforeRetries * sleepMultiplier);
1041 }
1042 }