diff options
author | Mukul Kumar Singh <msingh@apache.org> | 2018-05-18 14:08:46 +0530 |
---|---|---|
committer | Mukul Kumar Singh <msingh@apache.org> | 2018-05-18 14:08:46 +0530 |
commit | 6e996867f641b297e3f88068f6d185b709d509b0 (patch) | |
tree | c70ce7657c7b870df7d3e00ac7d5821886be38bc /hadoop-hdds/container-service/src/main/java/org/apache | |
parent | 3159bffce23abf35754da2d7d51de7d8c2631ae3 (diff) |
HDDS-76. Modify SCMStorageReportProto to include the data dir paths as well as the StorageType info. Contributed by Shashikant Banerjee.
Diffstat (limited to 'hadoop-hdds/container-service/src/main/java/org/apache')
4 files changed, 258 insertions, 13 deletions
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java index e0e826c471..5f5b81f480 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java @@ -108,12 +108,14 @@ public class ContainerLocationManagerImpl implements ContainerLocationManager, @Override public StorageLocationReport[] getLocationReport() throws IOException { + boolean failed; StorageLocationReport[] reports = new StorageLocationReport[dataLocations.size()]; for (int idx = 0; idx < dataLocations.size(); idx++) { ContainerStorageLocation loc = dataLocations.get(idx); long scmUsed = 0; long remaining = 0; + failed = false; try { scmUsed = loc.getScmUsed(); remaining = loc.getAvailable(); @@ -123,13 +125,19 @@ public class ContainerLocationManagerImpl implements ContainerLocationManager, // reset scmUsed and remaining if df/du failed. scmUsed = 0; remaining = 0; + failed = true; } - // TODO: handle failed storage - // For now, include storage report for location that failed to get df/du. - StorageLocationReport r = new StorageLocationReport( - loc.getStorageUuId(), false, loc.getCapacity(), - scmUsed, remaining); + StorageLocationReport.Builder builder = + StorageLocationReport.newBuilder(); + builder.setStorageLocation(loc.getStorageLocation()) + .setId(loc.getStorageUuId()) + .setFailed(failed) + .setCapacity(loc.getCapacity()) + .setRemaining(remaining) + .setScmUsed(scmUsed) + .setStorageType(loc.getStorageType()); + StorageLocationReport r = builder.build(); reports[idx] = r; } return reports; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java index 240beba6f3..039b4c3293 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java @@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; @@ -38,6 +39,8 @@ import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMNodeReport; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMStorageReport; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.StorageTypeProto; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.helpers.ContainerData; @@ -818,11 +821,7 @@ public class ContainerManagerImpl implements ContainerManager { SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder(); for (int i = 0; i < reports.length; i++) { SCMStorageReport.Builder srb = SCMStorageReport.newBuilder(); - nrb.addStorageReport(i, srb.setStorageUuid(reports[i].getId()) - .setCapacity(reports[i].getCapacity()) - .setScmUsed(reports[i].getScmUsed()) - .setRemaining(reports[i].getRemaining()) - .build()); + nrb.addStorageReport(reports[i].getProtoBufMessage()); } return nrb.build(); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java index 7293895295..7431baa9f2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CachingGetSpaceUsed; import org.apache.hadoop.fs.DF; import org.apache.hadoop.fs.GetSpaceUsed; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.io.IOUtils; @@ -121,6 +122,14 @@ public class ContainerStorageLocation { return scmUsage.getUsed(); } + public String getStorageLocation() { + return getNormalizedUri().getRawPath(); + } + + public StorageType getStorageType() { + return dataLocation.getStorageType(); + } + public void shutdown() { saveScmUsed(); scmUsedSaved = true; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java index 7ef91a91f7..a5ad6c2b5e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java @@ -18,26 +18,38 @@ package org.apache.hadoop.ozone.container.common.impl; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdds.protocol.proto. + StorageContainerDatanodeProtocolProtos.SCMStorageReport; +import org.apache.hadoop.hdds.protocol.proto. + StorageContainerDatanodeProtocolProtos.StorageTypeProto; + +import java.io.IOException; + /** * Storage location stats of datanodes that provide back store for containers. * */ public class StorageLocationReport { - public static final StorageLocationReport[] EMPTY_ARRAY = {}; private final String id; private final boolean failed; private final long capacity; private final long scmUsed; private final long remaining; + private final StorageType storageType; + private final String storageLocation; - public StorageLocationReport(String id, boolean failed, - long capacity, long scmUsed, long remaining) { + private StorageLocationReport(String id, boolean failed, long capacity, + long scmUsed, long remaining, StorageType storageType, + String storageLocation) { this.id = id; this.failed = failed; this.capacity = capacity; this.scmUsed = scmUsed; this.remaining = remaining; + this.storageType = storageType; + this.storageLocation = storageLocation; } public String getId() { @@ -60,4 +72,221 @@ public class StorageLocationReport { return remaining; } + public String getStorageLocation() { + return storageLocation; + } + + public StorageType getStorageType() { + return storageType; + } + + + private StorageTypeProto getStorageTypeProto() throws + IOException { + StorageTypeProto storageTypeProto; + switch (getStorageType()) { + case SSD: + storageTypeProto = StorageTypeProto.SSD; + break; + case DISK: + storageTypeProto = StorageTypeProto.DISK; + break; + case ARCHIVE: + storageTypeProto = StorageTypeProto.ARCHIVE; + break; + case PROVIDED: + storageTypeProto = StorageTypeProto.PROVIDED; + break; + case RAM_DISK: + storageTypeProto = StorageTypeProto.RAM_DISK; + break; + default: + throw new IOException("Illegal Storage Type specified"); + } + return storageTypeProto; + } + + private static StorageType getStorageType(StorageTypeProto proto) throws + IOException { + StorageType storageType; + switch (proto) { + case SSD: + storageType = StorageType.SSD; + break; + case DISK: + storageType = StorageType.DISK; + break; + case ARCHIVE: + storageType = StorageType.ARCHIVE; + break; + case PROVIDED: + storageType = StorageType.PROVIDED; + break; + case RAM_DISK: + storageType = StorageType.RAM_DISK; + break; + default: + throw new IOException("Illegal Storage Type specified"); + } + return storageType; + } + + /** + * Returns the SCMStorageReport protoBuf message for the Storage Location + * report. + * @return SCMStorageReport + * @throws IOException In case, the storage type specified is invalid. + */ + public SCMStorageReport getProtoBufMessage() throws IOException{ + SCMStorageReport.Builder srb = SCMStorageReport.newBuilder(); + return srb.setStorageUuid(getId()) + .setCapacity(getCapacity()) + .setScmUsed(getScmUsed()) + .setRemaining(getRemaining()) + .setStorageType(getStorageTypeProto()) + .setStorageLocation(getStorageLocation()) + .setFailed(isFailed()) + .build(); + } + + /** + * Returns the StorageLocationReport from the protoBuf message. + * @param report SCMStorageReport + * @return StorageLocationReport + * @throws IOException in case of invalid storage type + */ + + public static StorageLocationReport getFromProtobuf(SCMStorageReport report) + throws IOException { + StorageLocationReport.Builder builder = StorageLocationReport.newBuilder(); + builder.setId(report.getStorageUuid()) + .setStorageLocation(report.getStorageLocation()); + if (report.hasCapacity()) { + builder.setCapacity(report.getCapacity()); + } + if (report.hasScmUsed()) { + builder.setScmUsed(report.getScmUsed()); + } + if (report.hasStorageType()) { + builder.setStorageType(getStorageType(report.getStorageType())); + } + if (report.hasRemaining()) { + builder.setRemaining(report.getRemaining()); + } + + if (report.hasFailed()) { + builder.setFailed(report.getFailed()); + } + return builder.build(); + } + + /** + * Returns StorageLocation.Builder instance. + * + * @return StorageLocation.Builder + */ + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Builder class for building StorageLocationReport. + */ + public static class Builder { + private String id; + private boolean failed; + private long capacity; + private long scmUsed; + private long remaining; + private StorageType storageType; + private String storageLocation; + + /** + * Sets the storageId. + * + * @param id storageId + * @return StorageLocationReport.Builder + */ + public Builder setId(String id) { + this.id = id; + return this; + } + + /** + * Sets whether the volume failed or not. + * + * @param failed whether volume failed or not + * @return StorageLocationReport.Builder + */ + public Builder setFailed(boolean failed) { + this.failed = failed; + return this; + } + + /** + * Sets the capacity of volume. + * + * @param capacity capacity + * @return StorageLocationReport.Builder + */ + public Builder setCapacity(long capacity) { + this.capacity = capacity; + return this; + } + /** + * Sets the scmUsed Value. + * + * @param scmUsed storage space used by scm + * @return StorageLocationReport.Builder + */ + public Builder setScmUsed(long scmUsed) { + this.scmUsed = scmUsed; + return this; + } + + /** + * Sets the remaining free space value. + * + * @param remaining remaining free space + * @return StorageLocationReport.Builder + */ + public Builder setRemaining(long remaining) { + this.remaining = remaining; + return this; + } + + /** + * Sets the storageType. + * + * @param storageType type of the storage used + * @return StorageLocationReport.Builder + */ + public Builder setStorageType(StorageType storageType) { + this.storageType = storageType; + return this; + } + + /** + * Sets the storageLocation. + * + * @param storageLocation location of the volume + * @return StorageLocationReport.Builder + */ + public Builder setStorageLocation(String storageLocation) { + this.storageLocation = storageLocation; + return this; + } + + /** + * Builds and returns StorageLocationReport instance. + * + * @return StorageLocationReport + */ + public StorageLocationReport build() { + return new StorageLocationReport(id, failed, capacity, scmUsed, + remaining, storageType, storageLocation); + } + + } + } |