Java源码示例：org.apache.hadoop.util.PureJavaCrc32
示例1
private static void createIndexFile(File indexFile, Configuration conf)
throws IOException {
if (indexFile.exists()) {
System.out.println("Deleting existing file");
indexFile.delete();
}
indexFile.createNewFile();
FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
new Path(indexFile.getAbsolutePath()));
Checksum crc = new PureJavaCrc32();
crc.reset();
CheckedOutputStream chk = new CheckedOutputStream(output, crc);
String msg = "Writing new index file. This file will be used only " +
"for the testing.";
chk.write(Arrays.copyOf(msg.getBytes(),
MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
output.writeLong(chk.getChecksum().getValue());
output.close();
}
示例2
private static void createIndexFile(File indexFile, Configuration conf)
throws IOException {
if (indexFile.exists()) {
System.out.println("Deleting existing file");
indexFile.delete();
}
indexFile.createNewFile();
FSDataOutputStream output = FileSystem.getLocal(conf).getRaw().append(
new Path(indexFile.getAbsolutePath()));
Checksum crc = new PureJavaCrc32();
crc.reset();
CheckedOutputStream chk = new CheckedOutputStream(output, crc);
String msg = "Writing new index file. This file will be used only " +
"for the testing.";
chk.write(Arrays.copyOf(msg.getBytes(),
MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH));
output.writeLong(chk.getChecksum().getValue());
output.close();
}
示例3
public BlockXCodingMerger(Block block, int namespaceId,
DataInputStream[] childInputStreams, long offsetInBlock,
long length, String[] childAddrs, String myAddr,
DataTransferThrottler throttler,
int mergerLevel) throws IOException{
super();
this.block = block;
this.namespaceId = namespaceId;
this.childInputStreams = childInputStreams;
this.offsetInBlock = offsetInBlock;
this.length = length;
this.childAddrs = childAddrs;
this.myAddr = myAddr;
this.throttler = throttler;
this.mergerLevel = mergerLevel;
Configuration conf = new Configuration();
this.packetSize = conf.getInt("raid.blockreconstruct.packetsize", 4096);
this.bytesPerChecksum = conf.getInt("io.bytes.per.checksum", 512);
this.checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
bytesPerChecksum, new PureJavaCrc32());
this.checksumSize = checksum.getChecksumSize();
}
示例4
public static void outputRecords(OutputStream out,
boolean useAscii,
Unsigned16 firstRecordNumber,
Unsigned16 recordsToGenerate,
Unsigned16 checksum
) throws IOException {
byte[] row = new byte[100];
Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
Checksum crc = new PureJavaCrc32();
Unsigned16 tmp = new Unsigned16();
lastRecordNumber.add(recordsToGenerate);
Unsigned16 ONE = new Unsigned16(1);
Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
while (!recordNumber.equals(lastRecordNumber)) {
Random16.nextRand(rand);
if (useAscii) {
generateAsciiRecord(row, rand, recordNumber);
} else {
generateRecord(row, rand, recordNumber);
}
if (checksum != null) {
crc.reset();
crc.update(row, 0, row.length);
tmp.set(crc.getValue());
checksum.add(tmp);
}
recordNumber.add(ONE);
out.write(row);
}
}
示例5
public static void outputRecords(OutputStream out,
boolean useAscii,
Unsigned16 firstRecordNumber,
Unsigned16 recordsToGenerate,
Unsigned16 checksum
) throws IOException {
byte[] row = new byte[100];
Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
Checksum crc = new PureJavaCrc32();
Unsigned16 tmp = new Unsigned16();
lastRecordNumber.add(recordsToGenerate);
Unsigned16 ONE = new Unsigned16(1);
Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
while (!recordNumber.equals(lastRecordNumber)) {
Random16.nextRand(rand);
if (useAscii) {
generateAsciiRecord(row, rand, recordNumber);
} else {
generateRecord(row, rand, recordNumber);
}
if (checksum != null) {
crc.reset();
crc.update(row, 0, row.length);
tmp.set(crc.getValue());
checksum.add(tmp);
}
recordNumber.add(ONE);
out.write(row);
}
}
示例6
public static void outputRecords(OutputStream out,
boolean useAscii,
Unsigned16 firstRecordNumber,
Unsigned16 recordsToGenerate,
Unsigned16 checksum
) throws IOException {
byte[] row = new byte[100];
Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
Checksum crc = new PureJavaCrc32();
Unsigned16 tmp = new Unsigned16();
lastRecordNumber.add(recordsToGenerate);
Unsigned16 ONE = new Unsigned16(1);
Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
while (!recordNumber.equals(lastRecordNumber)) {
Random16.nextRand(rand);
if (useAscii) {
generateAsciiRecord(row, rand, recordNumber);
} else {
generateRecord(row, rand, recordNumber);
}
if (checksum != null) {
crc.reset();
crc.update(row, 0, row.length);
tmp.set(crc.getValue());
checksum.add(tmp);
}
recordNumber.add(ONE);
out.write(row);
}
}
示例7
/**
* Create a checksum output stream that writes
* the bytes to the given stream.
* @param out
*/
public IFileOutputStream(OutputStream out) {
super(out);
sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
Integer.MAX_VALUE, new PureJavaCrc32());
barray = new byte[sum.getChecksumSize()];
}
示例8
/**
* Create a checksum input stream that reads
* @param in The input stream to be verified for checksum.
* @param len The length of the input stream including checksum bytes.
*/
public IFileInputStream(InputStream in, long len) {
this.in = in;
sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
Integer.MAX_VALUE, new PureJavaCrc32());
checksumSize = sum.getChecksumSize();
length = len;
dataLength = length - checksumSize;
}
示例9
private DFSOutputStream(DFSClient dfsClient, String src, long blockSize,
Progressable progress, int bytesPerChecksum, short replication, boolean forceSync,
boolean doParallelWrites, DatanodeInfo[] favoredNodes)
throws IOException {
super(new CRC32(), bytesPerChecksum, 4);
this.dfsClient = dfsClient;
this.forceSync = forceSync;
this.doParallelWrites = doParallelWrites;
this.src = src;
this.blockSize = blockSize;
this.blockReplication = replication;
this.progress = progress;
streamer = new DataStreamer();
packetTimeout =
dfsClient.conf.getLong("dfs.client.packet.timeout", 15000); // 15 seconds
// try block recovery 5 times:
maxRecoveryErrorCount =
dfsClient.conf.getInt("dfs.client.block.recovery.retries", 5);
if (progress != null) {
DFSClient.LOG.debug("Set non-null progress callback on DFSOutputStream "+src);
}
this.favoredNodes = favoredNodes;
if ( bytesPerChecksum < 1 || blockSize % bytesPerChecksum != 0) {
throw new IOException("io.bytes.per.checksum(" + bytesPerChecksum +
") and blockSize(" + blockSize +
") do not match. " + "blockSize should be a " +
"multiple of io.bytes.per.checksum");
}
checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32,
bytesPerChecksum,
new PureJavaCrc32());
}
示例10
/**
* object constructor
*/
public BlockReaderAccelerator(
Configuration conf,
InetSocketAddress targetAddress,
DatanodeInfo chosenNode,
int dataTransferVersion,
int namespaceId,
String clientName,
Socket sock,
String hdfsfile,
LocatedBlock blk,
long startOffset,
long length,
boolean verifyChecksum,
DFSClientMetrics metrics) throws IOException {
this.conf = conf;
this.targetAddress = targetAddress;
this.datanodeInfo = chosenNode;
this.dataTransferVersion = dataTransferVersion;
this.namespaceId = namespaceId;
this.clientName = clientName;
this.sock = sock;
this.hdfsfile = hdfsfile;
this.blk = blk;
this.startOffset = startOffset;
this.length = length;
this.verifyChecksum = verifyChecksum;
this.metrics = metrics;
// create a checksum checker
if (this.verifyChecksum) {
this.checker = new PureJavaCrc32();
}
}
示例11
public static void outputRecords(OutputStream out,
boolean useAscii,
Unsigned16 firstRecordNumber,
Unsigned16 recordsToGenerate,
Unsigned16 checksum
) throws IOException {
byte[] row = new byte[100];
Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
Checksum crc = new PureJavaCrc32();
Unsigned16 tmp = new Unsigned16();
lastRecordNumber.add(recordsToGenerate);
Unsigned16 ONE = new Unsigned16(1);
Unsigned16 rand = Random16.skipAhead(firstRecordNumber);
while (!recordNumber.equals(lastRecordNumber)) {
Random16.nextRand(rand);
if (useAscii) {
generateAsciiRecord(row, rand, recordNumber);
} else {
generateRecord(row, rand, recordNumber);
}
if (checksum != null) {
crc.reset();
crc.update(row, 0, row.length);
tmp.set(crc.getValue());
checksum.add(tmp);
}
recordNumber.add(ONE);
out.write(row);
}
}
示例12
private static void createIndexFile(File indexFile, Configuration conf)
throws IOException {
if (indexFile.exists()) {
System.out.println("Deleting existing file");
indexFile.delete();
}
Checksum crc = new PureJavaCrc32();
TezSpillRecord tezSpillRecord = new TezSpillRecord(2);
tezSpillRecord.putIndex(new TezIndexRecord(0, 10, 10), 0);
tezSpillRecord.putIndex(new TezIndexRecord(10, 10, 10), 1);
tezSpillRecord.writeToFile(new Path(indexFile.getAbsolutePath()), conf,
FileSystem.getLocal(conf).getRaw(), crc);
}
示例13
@Test
public void testPureJavaCrc32ByteBuffer() {
final Checksum expected = new PureJavaCrc32();
final ChecksumByteBuffer testee = new PureJavaCrc32ByteBuffer();
new VerifyChecksumByteBuffer(expected, testee).testCorrectness();
}
示例14
public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner)
throws IOException {
this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
示例15
/**
* Write this spill record to the location provided.
*/
public void writeToFile(Path loc, JobConf job)
throws IOException {
writeToFile(loc, job, new PureJavaCrc32());
}
示例16
public SpillRecord(Path indexFileName, JobConf job, String expectedIndexOwner)
throws IOException {
this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
示例17
/**
* Write this spill record to the location provided.
*/
public void writeToFile(Path loc, JobConf job)
throws IOException {
writeToFile(loc, job, new PureJavaCrc32());
}
示例18
public SpillRecord(Path indexFileName, JobConf job) throws IOException {
this(indexFileName, job, new PureJavaCrc32());
}
示例19
/**
* Write this spill record to the location provided.
*/
public void writeToFile(Path loc, JobConf job)
throws IOException {
writeToFile(loc, job, new PureJavaCrc32());
}
示例20
public static BlockReader newBlockReader( int dataTransferVersion,
int namespaceId,
Socket sock, String file,
long blockId,
long genStamp,
long startOffset, long len,
int bufferSize, boolean verifyChecksum,
String clientName, long minSpeedBps)
throws IOException {
// in and out will be closed when sock is closed (by the caller)
DataOutputStream out = new DataOutputStream(
new BufferedOutputStream(NetUtils.getOutputStream(sock,HdfsConstants.WRITE_TIMEOUT)));
//write the header.
ReadBlockHeader readBlockHeader = new ReadBlockHeader(
dataTransferVersion, namespaceId, blockId, genStamp, startOffset, len,
clientName);
readBlockHeader.writeVersionAndOpCode(out);
readBlockHeader.write(out);
out.flush();
//
// Get bytes in block, set streams
//
DataInputStream in = new DataInputStream(
new BufferedInputStream(NetUtils.getInputStream(sock),
bufferSize));
if ( in.readShort() != DataTransferProtocol.OP_STATUS_SUCCESS ) {
throw new IOException("Got error in response to OP_READ_BLOCK " +
"self=" + sock.getLocalSocketAddress() +
", remote=" + sock.getRemoteSocketAddress() +
" for file " + file +
" for block " + blockId);
}
DataChecksum checksum = DataChecksum.newDataChecksum( in , new PureJavaCrc32());
//Warning when we get CHECKSUM_NULL?
// Read the first chunk offset.
long firstChunkOffset = in.readLong();
if ( firstChunkOffset < 0 || firstChunkOffset > startOffset ||
firstChunkOffset >= (startOffset + checksum.getBytesPerChecksum())) {
throw new IOException("BlockReader: error in first chunk offset (" +
firstChunkOffset + ") startOffset is " +
startOffset + " for file " + file);
}
return new BlockReader(file, blockId, in, checksum, verifyChecksum,
startOffset, firstChunkOffset, sock, minSpeedBps, dataTransferVersion);
}
示例21
protected Checksum initialValue() {
return new PureJavaCrc32();
}
示例22
protected Checksum initialValue() {
return new PureJavaCrc32();
}
示例23
public TezSpillRecord(Path indexFileName, Configuration job, String expectedIndexOwner)
throws IOException {
this(indexFileName, job, new PureJavaCrc32(), expectedIndexOwner);
}
示例24
/**
* Write this spill record to the location provided.
*/
public void writeToFile(Path loc, Configuration job)
throws IOException {
writeToFile(loc, job, new PureJavaCrc32());
}
示例25
public TezSpillRecord(Path indexFileName, FileSystem fs, String expectedIndexOwner)
throws IOException {
this(indexFileName, fs, new PureJavaCrc32(), expectedIndexOwner);
}
示例26
/**
* Write this spill record to the location provided.
*/
public void writeToFile(Path loc, Configuration job, FileSystem fs) throws IOException {
writeToFile(loc, job, fs, new PureJavaCrc32());
}
