You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
parallel-library/cps/static/js/archive/unzip.js

661 lines
24 KiB
JavaScript

/**
* unzip.js
*
* Licensed under the MIT License
*
* Copyright(c) 2011 Google Inc.
* Copyright(c) 2011 antimatter15
*
* Reference Documentation:
*
* ZIP format: http://www.pkware.com/documents/casestudies/APPNOTE.TXT
* DEFLATE format: http://tools.ietf.org/html/rfc1951
*/
/* global bitjs, importScripts, Uint8Array*/
// This file expects to be invoked as a Worker (see onmessage below).
importScripts("../io/bitstream.js");
importScripts("../io/bytebuffer.js");
importScripts("../io/bytestream.js");
importScripts("archive.js");
// Progress variables.
var currentFilename = "";
var currentFileNumber = 0;
var currentBytesUnarchivedInFile = 0;
var currentBytesUnarchived = 0;
var totalUncompressedBytesInArchive = 0;
var totalFilesInArchive = 0;
// Helper functions.
var info = function(str) {
postMessage(new bitjs.archive.UnarchiveInfoEvent(str));
};
var err = function(str) {
postMessage(new bitjs.archive.UnarchiveErrorEvent(str));
};
var postProgress = function() {
postMessage(new bitjs.archive.UnarchiveProgressEvent(
currentFilename,
currentFileNumber,
currentBytesUnarchivedInFile,
currentBytesUnarchived,
totalUncompressedBytesInArchive,
totalFilesInArchive));
};
var zLocalFileHeaderSignature = 0x04034b50;
var zArchiveExtraDataSignature = 0x08064b50;
var zCentralFileHeaderSignature = 0x02014b50;
var zDigitalSignatureSignature = 0x05054b50;
// takes a ByteStream and parses out the local file information
var ZipLocalFile = function(bstream) {
if (typeof bstream !== typeof {} || !bstream.readNumber || typeof bstream.readNumber !== typeof function() {}) {
return null;
}
bstream.readNumber(4); // swallow signature
this.version = bstream.readNumber(2);
this.generalPurpose = bstream.readNumber(2);
this.compressionMethod = bstream.readNumber(2);
this.lastModFileTime = bstream.readNumber(2);
this.lastModFileDate = bstream.readNumber(2);
this.crc32 = bstream.readNumber(4);
this.compressedSize = bstream.readNumber(4);
this.uncompressedSize = bstream.readNumber(4);
this.fileNameLength = bstream.readNumber(2);
this.extraFieldLength = bstream.readNumber(2);
this.filename = null;
if (this.fileNameLength > 0) {
this.filename = bstream.readString(this.fileNameLength);
}
this.extraField = null;
if (this.extraFieldLength > 0) {
this.extraField = bstream.readString(this.extraFieldLength);
info(" extra field=" + this.extraField);
}
// read in the compressed data
this.fileData = null;
if (this.compressedSize > 0) {
this.fileData = new Uint8Array(bstream.bytes.buffer, bstream.ptr, this.compressedSize);
bstream.ptr += this.compressedSize;
}
// TODO: deal with data descriptor if present (we currently assume no data descriptor!)
// "This descriptor exists only if bit 3 of the general purpose bit flag is set"
// But how do you figure out how big the file data is if you don't know the compressedSize
// from the header?!?
if ((this.generalPurpose & bitjs.BIT[3]) !== 0) {
this.crc32 = bstream.readNumber(4);
this.compressedSize = bstream.readNumber(4);
this.uncompressedSize = bstream.readNumber(4);
}
// Now that we have all the bytes for this file, we can print out some information.
info("Zip Local File Header:");
info(" version=" + this.version);
info(" general purpose=" + this.generalPurpose);
info(" compression method=" + this.compressionMethod);
info(" last mod file time=" + this.lastModFileTime);
info(" last mod file date=" + this.lastModFileDate);
info(" crc32=" + this.crc32);
info(" compressed size=" + this.compressedSize);
info(" uncompressed size=" + this.uncompressedSize);
info(" file name length=" + this.fileNameLength);
info(" extra field length=" + this.extraFieldLength);
info(" filename = '" + this.filename + "'");
};
// determine what kind of compressed data we have and decompress
ZipLocalFile.prototype.unzip = function() {
// Zip Version 1.0, no compression (store only)
if (this.compressionMethod === 0 ) {
info("ZIP v" + this.version + ", store only: " + this.filename + " (" + this.compressedSize + " bytes)");
currentBytesUnarchivedInFile = this.compressedSize;
currentBytesUnarchived += this.compressedSize;
this.fileData = zeroCompression(this.fileData, this.uncompressedSize);
} else if (this.compressionMethod === 8) {
// version == 20, compression method == 8 (DEFLATE)
info("ZIP v2.0, DEFLATE: " + this.filename + " (" + this.compressedSize + " bytes)");
this.fileData = inflate(this.fileData, this.uncompressedSize);
} else {
err("UNSUPPORTED VERSION/FORMAT: ZIP v" + this.version + ", compression method=" + this.compressionMethod + ": " + this.filename + " (" + this.compressedSize + " bytes)");
this.fileData = null;
}
};
// Takes an ArrayBuffer of a zip file in
// returns null on error
// returns an array of DecompressedFile objects on success
// ToDo This function differs
var unzip = function(arrayBuffer) {
postMessage(new bitjs.archive.UnarchiveStartEvent());
currentFilename = "";
currentFileNumber = 0;
currentBytesUnarchivedInFile = 0;
currentBytesUnarchived = 0;
totalUncompressedBytesInArchive = 0;
totalFilesInArchive = 0;
currentBytesUnarchived = 0;
var bstream = new bitjs.io.ByteStream(arrayBuffer);
// detect local file header signature or return null
if (bstream.peekNumber(4) === zLocalFileHeaderSignature) {
var localFiles = [];
// loop until we don't see any more local files
while (bstream.peekNumber(4) === zLocalFileHeaderSignature) {
var oneLocalFile = new ZipLocalFile(bstream);
// this should strip out directories/folders
if (oneLocalFile && oneLocalFile.uncompressedSize > 0 && oneLocalFile.fileData) {
localFiles.push(oneLocalFile);
totalUncompressedBytesInArchive += oneLocalFile.uncompressedSize;
}
}
totalFilesInArchive = localFiles.length;
// got all local files, now sort them
localFiles.sort(alphanumCase);
// archive extra data record
if (bstream.peekNumber(4) === zArchiveExtraDataSignature) {
info(" Found an Archive Extra Data Signature");
// skipping this record for now
bstream.readNumber(4);
var archiveExtraFieldLength = bstream.readNumber(4);
bstream.readString(archiveExtraFieldLength);
}
// central directory structure
// TODO: handle the rest of the structures (Zip64 stuff)
if (bstream.peekNumber(4) === zCentralFileHeaderSignature) {
info(" Found a Central File Header");
// read all file headers
while (bstream.peekNumber(4) === zCentralFileHeaderSignature) {
bstream.readNumber(4); // signature
bstream.readNumber(2); // version made by
bstream.readNumber(2); // version needed to extract
bstream.readNumber(2); // general purpose bit flag
bstream.readNumber(2); // compression method
bstream.readNumber(2); // last mod file time
bstream.readNumber(2); // last mod file date
bstream.readNumber(4); // crc32
bstream.readNumber(4); // compressed size
bstream.readNumber(4); // uncompressed size
var fileNameLength = bstream.readNumber(2); // file name length
var extraFieldLength = bstream.readNumber(2); // extra field length
var fileCommentLength = bstream.readNumber(2); // file comment length
bstream.readNumber(2); // disk number start
bstream.readNumber(2); // internal file attributes
bstream.readNumber(4); // external file attributes
bstream.readNumber(4); // relative offset of local header
bstream.readString(fileNameLength); // file name
bstream.readString(extraFieldLength); // extra field
bstream.readString(fileCommentLength); // file comment
}
}
// digital signature
if (bstream.peekNumber(4) === zDigitalSignatureSignature) {
info(" Found a Digital Signature");
bstream.readNumber(4);
var sizeOfSignature = bstream.readNumber(2);
bstream.readString(sizeOfSignature); // digital signature data
}
// report # files and total length
if (localFiles.length > 0) {
postProgress();
}
// now do the unzipping of each file
for (var i = 0; i < localFiles.length; ++i) {
var localfile = localFiles[i];
// update progress
currentFilename = localfile.filename;
currentFileNumber = i;
currentBytesUnarchivedInFile = 0;
// actually do the unzipping
localfile.unzip();
if (localfile.fileData !== null) {
postMessage(new bitjs.archive.UnarchiveExtractEvent(localfile));
postProgress();
}
}
postProgress();
postMessage(new bitjs.archive.UnarchiveFinishEvent());
}
};
// returns a table of Huffman codes
// each entry's index is its code and its value is a JavaScript object
// containing {length: 6, symbol: X}
function getHuffmanCodes(bitLengths) {
// ensure bitLengths is an array containing at least one element
if (typeof bitLengths !== typeof [] || bitLengths.length < 1) {
err("Error! getHuffmanCodes() called with an invalid array");
return null;
}
// Reference: http://tools.ietf.org/html/rfc1951#page-8
var numLengths = bitLengths.length;
var blCount = [];
var MAX_BITS = 1;
// Step 1: count up how many codes of each length we have
for (var i = 0; i < numLengths; ++i) {
var length = bitLengths[i];
// test to ensure each bit length is a positive, non-zero number
if (typeof length !== typeof 1 || length < 0) {
err("bitLengths contained an invalid number in getHuffmanCodes(): " + length + " of type " + (typeof length));
return null;
}
// increment the appropriate bitlength count
if (typeof blCount[length] === "undefined") blCount[length] = 0;
// a length of zero means this symbol is not participating in the huffman coding
if (length > 0) blCount[length]++;
if (length > MAX_BITS) MAX_BITS = length;
}
// Step 2: Find the numerical value of the smallest code for each code length
var nextCode = [];
var code = 0;
for (var bits = 1; bits <= MAX_BITS; ++bits) {
var length2 = bits - 1;
// ensure undefined lengths are zero
if (typeof blCount[length2] === "undefined") blCount[length2] = 0;
code = (code + blCount[bits - 1]) << 1;
nextCode [bits] = code;
}
// Step 3: Assign numerical values to all codes
var table = {};
var tableLength = 0;
for (var n = 0; n < numLengths; ++n) {
var len = bitLengths[n];
if (len !== 0) {
table[nextCode [len]] = { length: len, symbol: n }; //, bitstring: binaryValueToString(nextCode [len],len) };
tableLength++;
nextCode [len]++;
}
}
table.maxLength = tableLength;
return table;
}
/*
The Huffman codes for the two alphabets are fixed, and are not
represented explicitly in the data. The Huffman code lengths
for the literal/length alphabet are:
Lit Value Bits Codes
--------- ---- -----
0 - 143 8 00110000 through
10111111
144 - 255 9 110010000 through
111111111
256 - 279 7 0000000 through
0010111
280 - 287 8 11000000 through
11000111
*/
// fixed Huffman codes go from 7-9 bits, so we need an array whose index can hold up to 9 bits
var fixedHCtoLiteral = null;
var fixedHCtoDistance = null;
function getFixedLiteralTable() {
// create once
if (!fixedHCtoLiteral) {
var bitlengths = new Array(288);
var i;
for (i = 0; i <= 143; ++i) bitlengths[i] = 8;
for (i = 144; i <= 255; ++i) bitlengths[i] = 9;
for (i = 256; i <= 279; ++i) bitlengths[i] = 7;
for (i = 280; i <= 287; ++i) bitlengths[i] = 8;
// get huffman code table
fixedHCtoLiteral = getHuffmanCodes(bitlengths);
}
return fixedHCtoLiteral;
}
function getFixedDistanceTable() {
// create once
if (!fixedHCtoDistance) {
var bitlengths = new Array(32);
for (var i = 0; i < 32; ++i) {
bitlengths[i] = 5;
}
// get huffman code table
fixedHCtoDistance = getHuffmanCodes(bitlengths);
}
return fixedHCtoDistance;
}
// extract one bit at a time until we find a matching Huffman Code
// then return that symbol
function decodeSymbol(bstream, hcTable) {
var code = 0;
var len = 0;
// loop until we match
for (;;) {
// read in next bit
var bit = bstream.readBits(1);
code = (code << 1) | bit;
++len;
// check against Huffman Code table and break if found
if (hcTable.hasOwnProperty(code) && hcTable[code].length === len) {
break;
}
if (len > hcTable.maxLength) {
err("Bit stream out of sync, didn't find a Huffman Code, length was " + len +
" and table only max code length of " + hcTable.maxLength);
break;
}
}
return hcTable[code].symbol;
}
var CodeLengthCodeOrder = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15];
/*
Extra Extra Extra
Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
---- ---- ------ ---- ---- ------- ---- ---- -------
257 0 3 267 1 15,16 277 4 67-82
258 0 4 268 1 17,18 278 4 83-98
259 0 5 269 2 19-22 279 4 99-114
260 0 6 270 2 23-26 280 4 115-130
261 0 7 271 2 27-30 281 5 131-162
262 0 8 272 2 31-34 282 5 163-194
263 0 9 273 3 35-42 283 5 195-226
264 0 10 274 3 43-50 284 5 227-257
265 1 11,12 275 3 51-58 285 0 258
266 1 13,14 276 3 59-66
*/
var LengthLookupTable = [
[0, 3],
[0, 4],
[0, 5],
[0, 6],
[0, 7],
[0, 8],
[0, 9],
[0, 10],
[1, 11],
[1, 13],
[1, 15],
[1, 17],
[2, 19],
[2, 23],
[2, 27],
[2, 31],
[3, 35],
[3, 43],
[3, 51],
[3, 59],
[4, 67],
[4, 83],
[4, 99],
[4, 115],
[5, 131],
[5, 163],
[5, 195],
[5, 227],
[0, 258]
];
/*
Extra Extra Extra
Code Bits Dist Code Bits Dist Code Bits Distance
---- ---- ---- ---- ---- ------ ---- ---- --------
0 0 1 10 4 33-48 20 9 1025-1536
1 0 2 11 4 49-64 21 9 1537-2048
2 0 3 12 5 65-96 22 10 2049-3072
3 0 4 13 5 97-128 23 10 3073-4096
4 1 5,6 14 6 129-192 24 11 4097-6144
5 1 7,8 15 6 193-256 25 11 6145-8192
6 2 9-12 16 7 257-384 26 12 8193-12288
7 2 13-16 17 7 385-512 27 12 12289-16384
8 3 17-24 18 8 513-768 28 13 16385-24576
9 3 25-32 19 8 769-1024 29 13 24577-32768
*/
var DistLookupTable = [
[0, 1],
[0, 2],
[0, 3],
[0, 4],
[1, 5],
[1, 7],
[2, 9],
[2, 13],
[3, 17],
[3, 25],
[4, 33],
[4, 49],
[5, 65],
[5, 97],
[6, 129],
[6, 193],
[7, 257],
[7, 385],
[8, 513],
[8, 769],
[9, 1025],
[9, 1537],
[10, 2049],
[10, 3073],
[11, 4097],
[11, 6145],
[12, 8193],
[12, 12289],
[13, 16385],
[13, 24577]
];
function inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer) {
/*
loop (until end of block code recognized)
decode literal/length value from input stream
if value < 256
copy value (literal byte) to output stream
otherwise
if value = end of block (256)
break from loop
otherwise (value = 257..285)
decode distance from input stream
move backwards distance bytes in the output
stream, and copy length bytes from this
position to the output stream.
*/
var blockSize = 0;
for (;;) {
var symbol = decodeSymbol(bstream, hcLiteralTable);
if (symbol < 256) {
// copy literal byte to output
buffer.insertByte(symbol);
blockSize++;
} else {
// end of block reached
if (symbol === 256) {
break;
} else {
var lengthLookup = LengthLookupTable[symbol - 257];
var length = lengthLookup[1] + bstream.readBits(lengthLookup[0]);
var distLookup = DistLookupTable[decodeSymbol(bstream, hcDistanceTable)];
var distance = distLookup[1] + bstream.readBits(distLookup[0]);
// now apply length and distance appropriately and copy to output
// TODO: check that backward distance < data.length?
// http://tools.ietf.org/html/rfc1951#page-11
// "Note also that the referenced string may overlap the current
// position; for example, if the last 2 bytes decoded have values
// X and Y, a string reference with <length = 5, distance = 2>
// adds X,Y,X,Y,X to the output stream."
//
// loop for each character
var ch = buffer.ptr - distance;
blockSize += length;
if (length > distance) {
var data = buffer.data;
while (length--) {
buffer.insertByte(data[ch++]);
}
} else {
buffer.insertBytes(buffer.data.subarray(ch, ch + length));
}
} // length-distance pair
} // length-distance pair or end-of-block
} // loop until we reach end of block
return blockSize;
}
function zeroCompression(compressedData, numDecompressedBytes) {
var bstream = new bitjs.io.BitStream(compressedData.buffer,
false /* rtl */,
compressedData.byteOffset,
compressedData.byteLength);
var buffer = new bitjs.io.ByteBuffer(numDecompressedBytes);
buffer.insertBytes(bstream.readBytes(numDecompressedBytes));
return buffer.data;
}
// {Uint8Array} compressedData A Uint8Array of the compressed file data.
// compression method 8
// deflate: http://tools.ietf.org/html/rfc1951
function inflate(compressedData, numDecompressedBytes) {
// Bit stream representing the compressed data.
var bstream = new bitjs.io.BitStream(compressedData.buffer,
false /* rtl */,
compressedData.byteOffset,
compressedData.byteLength);
var buffer = new bitjs.io.ByteBuffer(numDecompressedBytes);
var blockSize = 0;
// block format: http://tools.ietf.org/html/rfc1951#page-9
var bFinal = 0;
do {
bFinal = bstream.readBits(1);
var bType = bstream.readBits(2);
blockSize = 0;
// ++numBlocks;
// no compression
if (bType === 0) {
// skip remaining bits in this byte
while (bstream.bitPtr !== 0) bstream.readBits(1);
var len = bstream.readBits(16);
bstream.readBits(16);
// TODO: check if nlen is the ones-complement of len?
if (len > 0) buffer.insertBytes(bstream.readBytes(len));
blockSize = len;
} else if (bType === 1) {
// fixed Huffman codes
blockSize = inflateBlockData(bstream, getFixedLiteralTable(), getFixedDistanceTable(), buffer);
} else if (bType === 2) {
// dynamic Huffman codes
var numLiteralLengthCodes = bstream.readBits(5) + 257;
var numDistanceCodes = bstream.readBits(5) + 1,
numCodeLengthCodes = bstream.readBits(4) + 4;
// populate the array of code length codes (first de-compaction)
var codeLengthsCodeLengths = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
for (var i = 0; i < numCodeLengthCodes; ++i) {
codeLengthsCodeLengths[ CodeLengthCodeOrder[i] ] = bstream.readBits(3);
}
// get the Huffman Codes for the code lengths
var codeLengthsCodes = getHuffmanCodes(codeLengthsCodeLengths);
// now follow this mapping
/*
0 - 15: Represent code lengths of 0 - 15
16: Copy the previous code length 3 - 6 times.
The next 2 bits indicate repeat length
(0 = 3, ... , 3 = 6)
Example: Codes 8, 16 (+2 bits 11),
16 (+2 bits 10) will expand to
12 code lengths of 8 (1 + 6 + 5)
17: Repeat a code length of 0 for 3 - 10 times.
(3 bits of length)
18: Repeat a code length of 0 for 11 - 138 times
(7 bits of length)
*/
// to generate the true code lengths of the Huffman Codes for the literal
// and distance tables together
var literalCodeLengths = [];
var prevCodeLength = 0;
while (literalCodeLengths.length < numLiteralLengthCodes + numDistanceCodes) {
var symbol = decodeSymbol(bstream, codeLengthsCodes);
if (symbol <= 15) {
literalCodeLengths.push(symbol);
prevCodeLength = symbol;
} else if (symbol === 16) {
var repeat = bstream.readBits(2) + 3;
while (repeat--) {
literalCodeLengths.push(prevCodeLength);
}
} else if (symbol === 17) {
var repeat1 = bstream.readBits(3) + 3;
while (repeat1--) {
literalCodeLengths.push(0);
}
} else if (symbol === 18) {
var repeat2 = bstream.readBits(7) + 11;
while (repeat2--) {
literalCodeLengths.push(0);
}
}
}
// now split the distance code lengths out of the literal code array
var distanceCodeLengths = literalCodeLengths.splice(numLiteralLengthCodes, numDistanceCodes);
// now generate the true Huffman Code tables using these code lengths
var hcLiteralTable = getHuffmanCodes(literalCodeLengths);
var hcDistanceTable = getHuffmanCodes(distanceCodeLengths);
blockSize = inflateBlockData(bstream, hcLiteralTable, hcDistanceTable, buffer);
} else {
// error
err("Error! Encountered deflate block of type 3");
return null;
}
// update progress
currentBytesUnarchivedInFile += blockSize;
currentBytesUnarchived += blockSize;
postProgress();
} while (bFinal !== 1);
// we are done reading blocks if the bFinal bit was set for this block
// return the buffer data bytes
return buffer.data;
}
// event.data.file has the ArrayBuffer.
onmessage = function(event) {
unzip(event.data.file, true);
};