Skip to content

Commit

Permalink
HDDS-11699. Wrote unit and integration tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@juncevich
juncevich committed Dec 25, 2024
1 parent fbfa61f commit ce79953
Show file tree
Hide file tree
Showing 3 changed files with 291 additions and 26 deletions.
155 changes: 130 additions & 25 deletions ...c/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneKeyLocation;
import org.apache.hadoop.ozone.client.OzoneMultipartUpload;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
Expand Down Expand Up @@ -179,12 +181,12 @@ public void preTest() throws Exception {
@Test
public void testInitiateMultipartUploadWithReplicationInformationSet() throws
IOException {
String uploadID = initiateMultipartUpload(bucket, keyName,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName,
ReplicationType.RATIS, ONE);

// Call initiate multipart upload for the same key again, this should
// generate a new uploadID.
String uploadIDNew = initiateMultipartUpload(bucket, keyName,
String uploadIDNew = initiateMultipartUploadWithAsserts(bucket, keyName,
ReplicationType.RATIS, ONE);
assertNotEquals(uploadIDNew, uploadID);
}
Expand Down Expand Up @@ -216,7 +218,7 @@ public void testInitiateMultipartUploadWithDefaultReplication() throws
@Test
public void testUploadPartWithNoOverride() throws IOException {
String sampleData = "sample Value";
String uploadID = initiateMultipartUpload(bucket, keyName,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName,
ReplicationType.RATIS, ONE);

OzoneOutputStream ozoneOutputStream = bucket.createMultipartKey(keyName,
Expand All @@ -235,7 +237,7 @@ public void testUploadPartWithNoOverride() throws IOException {
@Test
public void testUploadPartOverrideWithRatis() throws Exception {
String sampleData = "sample Value";
String uploadID = initiateMultipartUpload(bucket, keyName,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName,
ReplicationType.RATIS, THREE);

int partNumber = 1;
Expand Down Expand Up @@ -348,7 +350,7 @@ private OzoneBucket getOzoneECBucket(String myBucket)
@Test
public void testMultipartUploadWithPartsLessThanMinSize() throws Exception {
// Initiate multipart upload
String uploadID = initiateMultipartUpload(bucket, keyName, RATIS,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS,
ONE);

// Upload Parts
Expand All @@ -371,7 +373,7 @@ public void testMultipartUploadWithPartsLessThanMinSize() throws Exception {
public void testMultipartUploadWithDiscardedUnusedPartSize()
throws Exception {
// Initiate multipart upload
String uploadID = initiateMultipartUpload(bucket, keyName, RATIS, ONE);
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS, ONE);
byte[] data = generateData(10000000, (byte) 97);

// Upload Parts
Expand Down Expand Up @@ -402,7 +404,7 @@ public void testMultipartUploadWithDiscardedUnusedPartSize()
@Test
public void testMultipartUploadWithPartsMisMatchWithListSizeDifferent()
throws Exception {
String uploadID = initiateMultipartUpload(bucket, keyName, RATIS,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS,
ONE);

// We have not uploaded any parts, but passing some list it should throw
Expand All @@ -417,7 +419,7 @@ public void testMultipartUploadWithPartsMisMatchWithListSizeDifferent()
@Test
public void testMultipartUploadWithPartsMisMatchWithIncorrectPartName()
throws Exception {
String uploadID = initiateMultipartUpload(bucket, keyName, RATIS,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS,
ONE);

uploadPart(bucket, keyName, uploadID, 1, "data".getBytes(UTF_8));
Expand All @@ -432,7 +434,7 @@ public void testMultipartUploadWithPartsMisMatchWithIncorrectPartName()

@Test
public void testMultipartUploadWithMissingParts() throws Exception {
String uploadID = initiateMultipartUpload(bucket, keyName, RATIS,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS,
ONE);

uploadPart(bucket, keyName, uploadID, 1, "data".getBytes(UTF_8));
Expand All @@ -447,7 +449,7 @@ public void testMultipartUploadWithMissingParts() throws Exception {

@Test
public void testMultipartPartNumberExceedingAllowedRange() throws Exception {
String uploadID = initiateMultipartUpload(bucket, keyName,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName,
RATIS, ONE);
byte[] data = "data".getBytes(UTF_8);

Expand All @@ -469,7 +471,7 @@ public void testMultipartPartNumberExceedingAllowedRange() throws Exception {
public void testCommitPartAfterCompleteUpload() throws Exception {
String parentDir = "a/b/c/d/";
keyName = parentDir + UUID.randomUUID();
String uploadID = initiateMultipartUpload(bucket, keyName, RATIS, ONE);
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS, ONE);

assertEquals(volume.getBucket(bucketName).getUsedNamespace(), 4);

Expand Down Expand Up @@ -530,7 +532,7 @@ public void testAbortUploadFailWithInProgressPartUpload() throws Exception {
String parentDir = "a/b/c/d/";
keyName = parentDir + UUID.randomUUID();

String uploadID = initiateMultipartUpload(bucket, keyName,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName,
RATIS, ONE);

// Do not close output stream.
Expand All @@ -550,7 +552,7 @@ public void testAbortUploadSuccessWithOutAnyParts() throws Exception {
String parentDir = "a/b/c/d/";
keyName = parentDir + UUID.randomUUID();

String uploadID = initiateMultipartUpload(bucket, keyName, RATIS,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS,
ONE);
bucket.abortMultipartUpload(keyName, uploadID);
}
Expand All @@ -567,7 +569,7 @@ public void testAbortUploadSuccessWithParts() throws Exception {
ozoneManager.getMetadataManager().getBucketTable().get(buckKey);
BucketLayout bucketLayout = buckInfo.getBucketLayout();

String uploadID = initiateMultipartUpload(bucket, keyName, RATIS,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS,
ONE);
Pair<String, String> partNameAndETag = uploadPart(bucket, keyName, uploadID,
1, "data".getBytes(UTF_8));
Expand Down Expand Up @@ -599,7 +601,7 @@ public void testListMultipartUploadParts() throws Exception {
keyName = parentDir + "file-ABC";

Map<Integer, String> partsMap = new TreeMap<>();
String uploadID = initiateMultipartUpload(bucket, keyName, RATIS,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS,
ONE);
Pair<String, String> partNameAndETag1 = uploadPart(bucket, keyName,
uploadID, 1, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97));
Expand Down Expand Up @@ -688,7 +690,7 @@ private String verifyPartNames(Map<Integer, String> partsMap, int index,
public void testListMultipartUploadPartsWithContinuation()
throws Exception {
Map<Integer, String> partsMap = new TreeMap<>();
String uploadID = initiateMultipartUpload(bucket, keyName, RATIS,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS,
ONE);
Pair<String, String> partNameAndETag1 = uploadPart(bucket, keyName,
uploadID, 1, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97));
Expand Down Expand Up @@ -752,7 +754,7 @@ public void testListPartsWithInvalidInputs(int partNumberMarker, int maxParts, S
@Test
public void testListPartsWithPartMarkerGreaterThanPartCount()
throws Exception {
String uploadID = initiateMultipartUpload(bucket, keyName, RATIS,
String uploadID = initiateMultipartUploadWithAsserts(bucket, keyName, RATIS,
ONE);
uploadPart(bucket, keyName, uploadID, 1,
generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte)97));
Expand Down Expand Up @@ -795,11 +797,11 @@ public void testListMultipartUpload() throws Exception {
keys.add(key3);

// Initiate multipart upload
String uploadID1 = initiateMultipartUpload(bucket, key1, RATIS,
String uploadID1 = initiateMultipartUploadWithAsserts(bucket, key1, RATIS,
ONE);
String uploadID2 = initiateMultipartUpload(bucket, key2, RATIS,
String uploadID2 = initiateMultipartUploadWithAsserts(bucket, key2, RATIS,
ONE);
String uploadID3 = initiateMultipartUpload(bucket, key3, RATIS,
String uploadID3 = initiateMultipartUploadWithAsserts(bucket, key3, RATIS,
ONE);

// Upload Parts
Expand Down Expand Up @@ -854,6 +856,103 @@ public void testListMultipartUpload() throws Exception {
assertEquals(0, expectedList.size());
}

@Test
void testGetAllPartsWhenZeroPartNumber() throws Exception {
String parentDir = "a/b/c/d/e/f/";
keyName = parentDir + "file-ABC";
OzoneVolume s3volume = store.getVolume("s3v");
s3volume.createBucket(bucketName);
OzoneBucket s3Bucket = s3volume.getBucket(bucketName);

Map<Integer, String> partsMap = new TreeMap<>();
String uploadID = initiateMultipartUpload(s3Bucket, keyName, RATIS,
ONE);
Pair<String, String> partNameAndETag1 = uploadPart(s3Bucket, keyName,
uploadID, 1, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte) 97));
partsMap.put(1, partNameAndETag1.getKey());

Pair<String, String> partNameAndETag2 = uploadPart(s3Bucket, keyName,
uploadID, 2, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte) 97));
partsMap.put(2, partNameAndETag2.getKey());

Pair<String, String> partNameAndETag3 = uploadPart(s3Bucket, keyName,
uploadID, 3, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte) 97));
partsMap.put(3, partNameAndETag3.getKey());

s3Bucket.completeMultipartUpload(keyName, uploadID, partsMap);

OzoneKeyDetails s3KeyDetailsWithAllParts = ozClient.getProxy()
.getS3KeyDetails(s3Bucket.getName(), keyName, 0);
List<OzoneKeyLocation> ozoneKeyLocations = s3KeyDetailsWithAllParts.getOzoneKeyLocations();
assertEquals(6, ozoneKeyLocations.size());
}

@Test
void testGetParticularPart() throws Exception {
String parentDir = "a/b/c/d/e/f/";
keyName = parentDir + "file-ABC";
OzoneVolume s3volume = store.getVolume("s3v");
s3volume.createBucket(bucketName);
OzoneBucket s3Bucket = s3volume.getBucket(bucketName);

Map<Integer, String> partsMap = new TreeMap<>();
String uploadID = initiateMultipartUpload(s3Bucket, keyName, RATIS,
ONE);
Pair<String, String> partNameAndETag1 = uploadPart(s3Bucket, keyName,
uploadID, 1, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte) 97));
partsMap.put(1, partNameAndETag1.getKey());

Pair<String, String> partNameAndETag2 = uploadPart(s3Bucket, keyName,
uploadID, 2, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte) 97));
partsMap.put(2, partNameAndETag2.getKey());

Pair<String, String> partNameAndETag3 = uploadPart(s3Bucket, keyName,
uploadID, 3, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte) 97));
partsMap.put(3, partNameAndETag3.getKey());

s3Bucket.completeMultipartUpload(keyName, uploadID, partsMap);

OzoneKeyDetails s3KeyDetailsOneParts = ozClient.getProxy().getS3KeyDetails(bucketName, keyName, 1);
assertEquals(2, s3KeyDetailsOneParts.getOzoneKeyLocations().size());

OzoneKeyDetails s3KeyDetailsTwoParts = ozClient.getProxy().getS3KeyDetails(bucketName, keyName, 2);
assertEquals(2, s3KeyDetailsTwoParts.getOzoneKeyLocations().size());

OzoneKeyDetails s3KeyDetailsThreeParts = ozClient.getProxy().getS3KeyDetails(bucketName, keyName, 3);
assertEquals(2, s3KeyDetailsThreeParts.getOzoneKeyLocations().size());
}

@Test
void testGetNotExistedPart() throws Exception {
String parentDir = "a/b/c/d/e/f/";
keyName = parentDir + "file-ABC";
OzoneVolume s3volume = store.getVolume("s3v");
s3volume.createBucket(bucketName);
OzoneBucket s3Bucket = s3volume.getBucket(bucketName);

Map<Integer, String> partsMap = new TreeMap<>();
String uploadID = initiateMultipartUpload(s3Bucket, keyName, RATIS,
ONE);
Pair<String, String> partNameAndETag1 = uploadPart(s3Bucket, keyName,
uploadID, 1, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte) 97));
partsMap.put(1, partNameAndETag1.getKey());

Pair<String, String> partNameAndETag2 = uploadPart(s3Bucket, keyName,
uploadID, 2, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte) 97));
partsMap.put(2, partNameAndETag2.getKey());

Pair<String, String> partNameAndETag3 = uploadPart(s3Bucket, keyName,
uploadID, 3, generateData(OzoneConsts.OM_MULTIPART_MIN_SIZE, (byte) 97));
partsMap.put(3, partNameAndETag3.getKey());

s3Bucket.completeMultipartUpload(keyName, uploadID, partsMap);

OzoneKeyDetails s3KeyDetailsWithNotExistedParts = ozClient.getProxy()
.getS3KeyDetails(s3Bucket.getName(), keyName, 4);
List<OzoneKeyLocation> ozoneKeyLocations = s3KeyDetailsWithNotExistedParts.getOzoneKeyLocations();
assertEquals(0, ozoneKeyLocations.size());
}

private String verifyUploadedPart(String uploadID, String partName,
OMMetadataManager metadataMgr) throws IOException {
OzoneManager ozoneManager = cluster.getOzoneManager();
Expand Down Expand Up @@ -891,11 +990,10 @@ private String verifyUploadedPart(String uploadID, String partName,
return multipartKey;
}

private String initiateMultipartUpload(OzoneBucket oBucket, String kName,
ReplicationType replicationType, ReplicationFactor replicationFactor)
throws IOException {
OmMultipartInfo multipartInfo = oBucket.initiateMultipartUpload(kName,
replicationType, replicationFactor);
private String initiateMultipartUploadWithAsserts(
OzoneBucket oBucket, String kName, ReplicationType replicationType, ReplicationFactor replicationFactor
) throws IOException {
OmMultipartInfo multipartInfo = oBucket.initiateMultipartUpload(kName, replicationType, replicationFactor);

assertNotNull(multipartInfo);
String uploadID = multipartInfo.getUploadID();
Expand All @@ -907,6 +1005,13 @@ private String initiateMultipartUpload(OzoneBucket oBucket, String kName,
return uploadID;
}

private String initiateMultipartUpload(
OzoneBucket oBucket, String kName, ReplicationType replicationType, ReplicationFactor replicationFactor
) throws IOException {
OmMultipartInfo multipartInfo = oBucket.initiateMultipartUpload(kName, replicationType, replicationFactor);
return multipartInfo.getUploadID();
}

private Pair<String, String> uploadPart(OzoneBucket oBucket, String kName,
String uploadID, int partNumber,
byte[] data)
Expand Down
Loading

0 comments on commit ce79953

Please sign in to comment.