diff options
| author | Xiaoyu Yao <xyao@apache.org> | 2018-01-11 11:27:58 -0800 |
|---|---|---|
| committer | Owen O'Malley <omalley@apache.org> | 2018-04-26 05:36:04 -0700 |
| commit | 98eeabbdb3c06f4b60ea2a0525879bb797232f95 (patch) | |
| tree | b16ad4324875fbe2b6ad12b714dae0dd021db4eb /hadoop-hdfs-project/hadoop-hdfs/src/main | |
| parent | 0c48cb2cf2b5d933e43f3f226f4a6be0b7c43f78 (diff) | |
HDFS-12794. Ozone: Parallelize ChunkOutputSream Writes to container. Contributed by Shashikant Banerjee.
Diffstat (limited to 'hadoop-hdfs-project/hadoop-hdfs/src/main')
2 files changed, 19 insertions, 0 deletions
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java index 1830c71597..137f8f95d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java @@ -67,6 +67,11 @@ import java.io.IOException; import java.io.OutputStream; import java.util.List; +import static org.apache.hadoop.ozone.OzoneConfigKeys + .OZONE_OUTPUT_STREAM_BUFFER_SIZE_IN_MB; +import static org.apache.hadoop.ozone.OzoneConfigKeys + .OZONE_OUTPUT_STREAM_BUFFER_SIZE_DEFAULT; + /** * A {@link StorageHandler} implementation that distributes object storage * across the nodes of an HDFS cluster. @@ -86,6 +91,7 @@ public final class DistributedStorageHandler implements StorageHandler { private final boolean useRatis; private final OzoneProtos.ReplicationType type; private final OzoneProtos.ReplicationFactor factor; + private final long streamBufferSize; /** * Creates a new DistributedStorageHandler. @@ -127,6 +133,9 @@ public final class DistributedStorageHandler implements StorageHandler { chunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE); chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE; } + // streamBufferSize by default is set to default scm block size. + streamBufferSize = conf.getLong(OZONE_OUTPUT_STREAM_BUFFER_SIZE_IN_MB, + OZONE_OUTPUT_STREAM_BUFFER_SIZE_DEFAULT) * OzoneConsts.MB; } @Override @@ -418,6 +427,7 @@ public final class DistributedStorageHandler implements StorageHandler { .setRequestID(args.getRequestID()) .setType(xceiverClientManager.getType()) .setFactor(xceiverClientManager.getFactor()) + .setStreamBufferSize(streamBufferSize) .build(); groupOutputStream.addPreallocateBlocks( openKey.getKeyInfo().getLatestVersionLocations(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml index 31c3901a95..4df99f92d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml @@ -691,6 +691,15 @@ </description> </property> <property> + <name>ozone.output.stream.buffer.size.in.mb</name> + <value>256</value> + <tag>OZONE</tag> + <description> + The maximum size of the buffer allocated for the ozone output stream for + write. Default size is equals to scm block size. + </description> + </property> + <property> <name>ozone.scm.chunk.size</name> <value>16777216</value> <tag>OZONE, SCM, CONTAINER, PERFORMANCE</tag> |