HDDS-11511. Introduce metrics in deletion services of OM

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java

@@ -42,6 +42,7 @@
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.DeletingServiceMetrics;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.om.OmSnapshotManager;
 import org.apache.hadoop.ozone.om.OzoneManager;
@@ -109,6 +110,7 @@ public class TestDirectoryDeletingServiceWithFSO {
   private static String volumeName;
   private static String bucketName;
   private static OzoneClient client;
+  private static DeletingServiceMetrics metrics;
 
   @BeforeAll
   public static void init() throws Exception {
@@ -140,6 +142,7 @@ public static void init() throws Exception {
     conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5);
 
     fs = FileSystem.get(conf);
+    metrics = cluster.getOzoneManager().getDeletionMetrics();
   }
 
   @AfterAll
@@ -185,6 +188,7 @@ public void testDeleteEmptyDirectory() throws Exception {
     assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
     assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);
 
+    metrics.resetDirectoryMetrics();
     // Delete the appRoot, empty dir
     fs.delete(appRoot, true);
 
@@ -197,6 +201,8 @@ public void testDeleteEmptyDirectory() throws Exception {
     assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 1);
     assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
     assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);
+    assertEquals(1, metrics.getNumDirsPurged());
+    assertEquals(1, metrics.getNumDirsSentForPurge());
 
     try (TableIterator<?, ? extends Table.KeyValue<?, OmDirectoryInfo>>
         iterator = dirTable.iterator()) {
@@ -255,6 +261,7 @@ public void testDeleteWithLargeSubPathsThanBatchSize() throws Exception {
 
     long preRunCount = dirDeletingService.getRunCount().get();
 
+    metrics.resetDirectoryMetrics();
     // Delete the appRoot
     fs.delete(appRoot, true);
 
@@ -270,6 +277,14 @@ public void testDeleteWithLargeSubPathsThanBatchSize() throws Exception {
     assertSubPathsCount(dirDeletingService::getMovedDirsCount, 18);
     assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 19);
 
+    assertEquals(15, metrics.getNumSubFilesSentForPurge());
+    assertEquals(15, metrics.getNumSubFilesMovedToDeletedTable());
+    assertEquals(19, metrics.getNumDirsPurged());
+    assertEquals(19, metrics.getNumDirsSentForPurge());
+    assertEquals(18, metrics.getNumSubDirsMovedToDeletedDirTable());
+    assertEquals(18, metrics.getNumSubDirsSentForPurge());
+
+
     long elapsedRunCount = dirDeletingService.getRunCount().get() - preRunCount;
     assertThat(dirDeletingService.getRunCount().get()).isGreaterThan(1);
     // Ensure dir deleting speed, here provide a backup value for safe CI
@@ -308,6 +323,7 @@ public void testDeleteWithMultiLevels() throws Exception {
     assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
     assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0);
 
+    metrics.resetDirectoryMetrics();
     // Delete the rootDir, which should delete all keys.
     fs.delete(root, true);
 
@@ -321,7 +337,66 @@ public void testDeleteWithMultiLevels() throws Exception {
     assertSubPathsCount(dirDeletingService::getMovedFilesCount, 3);
     assertSubPathsCount(dirDeletingService::getMovedDirsCount, 2);
     assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 5);
+    assertEquals(5, metrics.getNumDirsSentForPurge());
+    assertEquals(5, metrics.getNumDirsPurged());
+    assertEquals(4, metrics.getNumSubDirsMovedToDeletedDirTable());
+    assertEquals(4, metrics.getNumSubDirsSentForPurge());
+    assertEquals(3, metrics.getNumSubFilesSentForPurge());
+    assertEquals(3, metrics.getNumSubFilesMovedToDeletedTable());
+
+    assertThat(dirDeletingService.getRunCount().get()).isGreaterThan(1);
+  }
+
+  /**
+   * Test to check the following scenario:
+   * A subdir gets marked for move in DirectoryDeletingService and
+   * marked for delete in AbstractKeyDeletingService#optimizeDirDeletesAndSubmitRequest.
+   */
+  @Test
+  public void testDeleteWithLessDirsButMultipleLevels() throws Exception {
+    Path root = new Path("/rootDir");
+    Path appRoot = new Path(root, "appRoot");
+    Path parent = new Path(appRoot, "parentDir");
+    fs.mkdirs(parent);
+    Path child = new Path(parent, "childFile");
+    ContractTestUtils.touch(fs, child);
+
+    Table<String, OmKeyInfo> deletedDirTable =
+        cluster.getOzoneManager().getMetadataManager().getDeletedDirTable();
+    Table<String, OmKeyInfo> keyTable =
+        cluster.getOzoneManager().getMetadataManager().getKeyTable(getFSOBucketLayout());
+    Table<String, OmDirectoryInfo> dirTable = cluster.getOzoneManager().getMetadataManager().getDirectoryTable();
+
+    DirectoryDeletingService dirDeletingService =
+        (DirectoryDeletingService) cluster.getOzoneManager().getKeyManager().getDirDeletingService();
+
+    // Before delete
+    assertTableRowCount(deletedDirTable, 0);
+    assertTableRowCount(dirTable, 3);
+    assertTableRowCount(keyTable, 1);
+
+    assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);
+    assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
+    assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0);
+
+    metrics.resetDirectoryMetrics();
+    fs.delete(appRoot, true);
+
+    // After delete
+    checkPath(appRoot);
+    assertTableRowCount(deletedDirTable, 0);
+    assertTableRowCount(keyTable, 0);
+    assertTableRowCount(dirTable, 1);
+    assertSubPathsCount(dirDeletingService::getMovedFilesCount, 1);
+    assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 2);
+    assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
 
+    assertEquals(2, metrics.getNumDirsSentForPurge());
+    assertEquals(2, metrics.getNumDirsPurged());
+    assertEquals(1, metrics.getNumSubDirsMovedToDeletedDirTable());
+    assertEquals(1, metrics.getNumSubDirsSentForPurge());
+    assertEquals(1, metrics.getNumSubFilesSentForPurge());
+    assertEquals(1, metrics.getNumSubFilesMovedToDeletedTable());
     assertThat(dirDeletingService.getRunCount().get()).isGreaterThan(1);
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java

@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+/**
+ * Class contains metrics related to the OM Deletion services.
+ */
+@Metrics(about = "Deletion Service Metrics", context = OzoneConsts.OZONE)
+public final class DeletingServiceMetrics {
+
+  public static final String METRICS_SOURCE_NAME =
+      DeletingServiceMetrics.class.getSimpleName();
+  private MetricsRegistry registry;
+
+  private DeletingServiceMetrics() {
+    this.registry = new MetricsRegistry(METRICS_SOURCE_NAME);
+  }
+
+  /**
+   * Creates and returns DeletingServiceMetrics instance.
+   *
+   * @return DeletingServiceMetrics
+   */
+  public static DeletingServiceMetrics create() {
+    return DefaultMetricsSystem.instance().register(METRICS_SOURCE_NAME,
+        "Metrics tracking the progress of deletion of directories and keys in the OM",
+        new DeletingServiceMetrics());
+  }
+  /**
+   * Unregister the metrics instance.
+   */
+  public static void unregister() {
+    DefaultMetricsSystem.instance().unregisterSource(METRICS_SOURCE_NAME);
+  }
+
+
+  /*
+   * Total directory deletion metrics across all iterations of DirectoryDeletingService since last restart.
+   */
+  @Metric("Total no. of deleted directories sent for purge")
+  private MutableGaugeLong numDirsSentForPurge;
+  @Metric("Total no. of sub-directories sent for purge")
+  private MutableGaugeLong numSubDirsSentForPurge;
+  @Metric("Total no. of sub-files sent for purge")
+  private MutableGaugeLong numSubFilesSentForPurge;
+
+  public void incrNumDirsSentForPurge(long dirDel) {
+    numDirsSentForPurge.incr(dirDel);
+  }
+
+  public void incrNumSubDirsSentForPurge(long delta) {
+    numSubDirsSentForPurge.incr(delta);
+  }
+
+  public void incrNumSubFilesSentForPurge(long delta) {
+    numSubFilesSentForPurge.incr(delta);
+  }
+
+  public void incrementDirectoryDeletionTotalMetrics(long dirDel, long dirMove, long filesMove) {
+    incrNumDirsSentForPurge(dirDel);
+    incrNumSubDirsSentForPurge(dirMove);
+    incrNumSubFilesSentForPurge(filesMove);
+  }
+
+  public long getNumDirsSentForPurge() {
+    return numDirsSentForPurge.value();
+  }
+  public long getNumSubDirsSentForPurge() {
+    return numSubDirsSentForPurge.value();
+  }
+  public long getNumSubFilesSentForPurge() {
+    return numSubFilesSentForPurge.value();
+  }
+
+  /*
+   * Total key deletion metrics across all iterations of KeyDeletingService since last restart.
+   */
+  @Metric("Total no. of keys processed")
+  private MutableGaugeLong numKeysProcessed;
+  @Metric("Total no. of deleted keys sent for purge")
+  private MutableGaugeLong numKeysSentForPurge;
+
+  public void incrNumKeysProcessed(long keysProcessed) {
+    this.numKeysProcessed.incr(keysProcessed);
+  }
+
+  public void incrNumKeysSentForPurge(long keysPurge) {
+    this.numKeysSentForPurge.incr(keysPurge);
+  }
+
+  /*
+   * Directory purge request metrics.
+   */
+  @Metric("Total no. of directories purged")
+  private MutableGaugeLong numDirsPurged;
+  @Metric("Total no. of subFiles moved to deletedTable")
+  private MutableGaugeLong numSubFilesMovedToDeletedTable;
+  @Metric("Total no. of subDirectories moved to deletedDirTable")
+  private MutableGaugeLong numSubDirsMovedToDeletedDirTable;
+
+  public void incrNumDirPurged(long dirPurged) {
+    this.numDirsPurged.incr(dirPurged);
+  }
+  public void incrNumSubFilesMoved(long subKeys) {
+    this.numSubFilesMovedToDeletedTable.incr(subKeys);
+  }
+  public void incrNumSubDirectoriesMoved(long subDirectories) {
+    this.numSubDirsMovedToDeletedDirTable.incr(subDirectories);
+  }
+  public long getNumDirsPurged() {
+    return numDirsPurged.value();
+  }
+  public long getNumSubFilesMovedToDeletedTable() {
+    return numSubFilesMovedToDeletedTable.value();
+  }
+  public long getNumSubDirsMovedToDeletedDirTable() {
+    return numSubDirsMovedToDeletedDirTable.value();
+  }
+
+  /*
+   * Key purge request metrics.
+   */
+  @Metric("Total no. of keys purged")
+  private MutableGaugeLong numKeysPurged;
+
+  public void incrNumKeysPurged(long keysPurged) {
+    this.numKeysPurged.incr(keysPurged);
+  }
+
+  @VisibleForTesting
+  public void resetDirectoryMetrics() {
+    numDirsPurged.set(0);
+    numSubFilesMovedToDeletedTable.set(0);
+    numSubDirsMovedToDeletedDirTable.set(0);
+    numDirsSentForPurge.set(0);
+    numSubDirsSentForPurge.set(0);
+    numSubFilesSentForPurge.set(0);
+  }
+
+}

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -402,6 +402,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   private OMHAMetrics omhaMetrics;
   private final ProtocolMessageMetrics<ProtocolMessageEnum>
       omClientProtocolMetrics;
+  private final DeletingServiceMetrics omDeletionMetrics;
   private OzoneManagerHttpServer httpServer;
   private final OMStorage omStorage;
   private ObjectName omInfoBeanName;
@@ -684,6 +685,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
 
     metrics = OMMetrics.create();
     perfMetrics = OMPerformanceMetrics.register();
+    omDeletionMetrics = DeletingServiceMetrics.create();
     // Get admin list
     omStarterUser = UserGroupInformation.getCurrentUser().getShortUserName();
     omAdmins = OzoneAdmins.getOzoneAdmins(omStarterUser, conf);
@@ -1652,6 +1654,9 @@ public OMMetrics getMetrics() {
   public OMPerformanceMetrics getPerfMetrics() {
     return perfMetrics;
   }
+  public DeletingServiceMetrics getDeletionMetrics() {
+    return omDeletionMetrics;
+  }
 
   /**
    * Start service.
@@ -2317,6 +2322,7 @@ public boolean stop() {
       if (omRatisSnapshotProvider != null) {
         omRatisSnapshotProvider.close();
       }
+      DeletingServiceMetrics.unregister();
       OMPerformanceMetrics.unregister();
       RatisDropwizardExports.clear(ratisMetricsMap, ratisReporterList);
       scmClient.close();

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java

@@ -33,7 +33,9 @@
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.om.OMMetrics;
 import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+import org.apache.hadoop.ozone.om.DeletingServiceMetrics;
 import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
+
 import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
 import org.apache.hadoop.ozone.om.OzoneManager;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -75,6 +77,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
     OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager();
     Map<String, OmKeyInfo> openKeyInfoMap = new HashMap<>();
     OMMetrics omMetrics = ozoneManager.getMetrics();
+    DeletingServiceMetrics deletingServiceMetrics = ozoneManager.getDeletionMetrics();
     OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
         getOmRequest());
     final SnapshotInfo fromSnapshotInfo;
@@ -98,6 +101,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
       return new OMDirectoriesPurgeResponseWithFSO(createErrorOMResponse(omResponse, e));
     }
     try {
+      int numSubDirMoved = 0, numSubFilesMoved = 0, numDirsDeleted = 0;
       for (OzoneManagerProtocolProtos.PurgePathRequest path : purgeRequests) {
         for (OzoneManagerProtocolProtos.KeyInfo key :
             path.getMarkDeletedSubDirsList()) {
@@ -111,6 +115,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
             lockSet.add(volBucketPair);
           }
           omMetrics.decNumKeys();
+          numSubDirMoved++;
           OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager,
               volumeName, bucketName);
           // bucketInfo can be null in case of delete volume or bucket
@@ -153,6 +158,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
           }
 
           omMetrics.decNumKeys();
+          numSubFilesMoved++;
           OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager,
               volumeName, bucketName);
           // bucketInfo can be null in case of delete volume or bucket
@@ -168,7 +174,14 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
             volBucketInfoMap.putIfAbsent(volBucketPair, omBucketInfo);
           }
         }
+        if (path.hasDeletedDir()) {
+          numDirsDeleted++;
+        }
       }
+      deletingServiceMetrics.incrNumSubDirectoriesMoved(numSubDirMoved);
+      deletingServiceMetrics.incrNumSubFilesMoved(numSubFilesMoved);
+      deletingServiceMetrics.incrNumDirPurged(numDirsDeleted);
+
       if (fromSnapshotInfo != null) {
         fromSnapshotInfo.setLastTransactionInfo(TransactionInfo.valueOf(context.getTermIndex()).toByteString());
         omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshotInfo.getTableKey()),