diff --git a/README.md b/README.md index e416e0c..42472ec 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Currently supported: - gzip - tgz - zip +- xz (requires optional dependency `lzma-native`) ## Install @@ -33,7 +34,7 @@ npm install compressing ### Compress a single file -Use gzip as an example, tar, tgz and zip is same as gzip. +Use gzip as an example; tar, tgz, zip, and xz are the same as gzip. __promise style__ @@ -235,6 +236,7 @@ Use this API to compress a single file. This is a convenient method, which wraps - tar.compressFile(source, dest, opts) - tgz.compressFile(source, dest, opts) - zip.compressFile(source, dest, opts) +- xz.compressFile(source, dest, opts) Params @@ -268,6 +270,7 @@ Use this API to uncompress a file. This is a convenient method, which wraps Unco - tgz.uncompress(source, dest, opts) - zip.uncompress(source, dest, opts) - gzip.uncompress(source, dest, opts) +- xz.uncompress(source, dest, opts) Params @@ -291,6 +294,7 @@ __Note: If you are not very familiar with streams, just use compressFile() API, - new tar.FileStream(opts) - new tgz.FileStream(opts) - new zip.FileStream(opts) +- new xz.FileStream(opts) Common params: @@ -315,6 +319,12 @@ Zip params: - opts.relativePath {String} - Adds a file from source into the compressed result file as opts.relativePath. Uncompression programs would extract the file from the compressed file as relativePath. If opts.source is a file path, opts.relativePath is optional, otherwise it's required. - opts.yazl {Object} - zip.FileStream compression uses [yazl](https://github.com/thejoshwolfe/yazl), pass this param to control the behavior of yazl. +XZ params: + +- opts.lzma - {Object} xz.FileStream uses lzma-native to compress, pass this param to control the behavior of lzma-native. + +__Note: xz compression/decompression requires the optional dependency `lzma-native`. If you try to use xz features without installing it, you'll get an error asking you to install it.__ + ### Stream The readable stream to compress anything as you need. @@ -355,11 +365,20 @@ __Constructor__ - new tar.UncompressStream(opts) - new tgz.UncompressStream(opts) - new zip.UncompressStream(opts) +- new xz.UncompressStream(opts) Common params: - opts.source {String|Buffer|Stream} - source to be uncompressed, could be a file path, buffer, or a readable stream. +Gzip params: + +- opts.zlib - {Object} gzip.UncompressStream uses zlib to uncompress, pass this param to control the behavior of zlib. + +XZ params: + +- opts.lzma - {Object} xz.UncompressStream uses lzma-native to uncompress, pass this param to control the behavior of lzma-native. + __CAUTION for zip.UncompressStream__ Due to the design of the .zip file format, it's impossible to interpret a .zip file without loading all data into memory. diff --git a/index.d.ts b/index.d.ts index e5f8504..fb5cbc4 100644 --- a/index.d.ts +++ b/index.d.ts @@ -199,3 +199,25 @@ export namespace zip { } } + +export namespace xz { + function compressFile(source: sourceType, dest: destType, opts?: any): Promise + function uncompress(source: sourceType, dest: destType, opts?: any): Promise + function decompress(source: sourceType, dest: destType, opts?: any): Promise + + export class FileStream extends ReadStream { + constructor(opts?: { + lzma?: object, + source: sourceType + }); + } + + export class UncompressStream extends WriteStream { + constructor(opts?: { + lzma?: object, + source: sourceType + }); + on(event: string, listener: (...args: any[]) => void): this + on(event: 'error', listener: (err: Error) => void): this + } +} diff --git a/index.js b/index.js index b104632..259e10b 100644 --- a/index.js +++ b/index.js @@ -4,3 +4,4 @@ exports.zip = require('./lib/zip'); exports.gzip = require('./lib/gzip'); exports.tar = require('./lib/tar'); exports.tgz = require('./lib/tgz'); +exports.xz = require('./lib/xz'); diff --git a/lib/xz/file_stream.js b/lib/xz/file_stream.js new file mode 100644 index 0000000..4ea0671 --- /dev/null +++ b/lib/xz/file_stream.js @@ -0,0 +1,54 @@ +'use strict'; + +const fs = require('fs'); +const lzma = require('lzma-native'); +const utils = require('../utils'); +const streamifier = require('streamifier'); +const stream = require('stream'); + +class XzFileStream extends stream.Transform { + constructor(opts) { + opts = opts || {}; + super(opts); + + const sourceType = utils.sourceType(opts.source); + const compressor = lzma.createCompressor(opts.lzma); + + compressor.on('error', err => this.emit('error', err)); + compressor.on('end', () => this.push(null)); + compressor.on('data', chunk => this.push(chunk)); + + if (sourceType === 'file') { + const stream = fs.createReadStream(opts.source, opts.fs); + stream.on('error', err => this.emit('error', err)); + stream.pipe(compressor); + return; + } + + if (sourceType === 'buffer') { + const stream = streamifier.createReadStream(opts.source, opts.streamifier); + stream.on('error', err => this.emit('error', err)); + stream.pipe(compressor); + return; + } + + if (sourceType === 'stream') { + opts.source.on('error', err => this.emit('error', err)); + opts.source.pipe(compressor); + return; + } + + // For streaming input + this.on('pipe', srcStream => { + srcStream.unpipe(this); + srcStream.pipe(compressor); + }); + } + + _transform(chunk, encoding, callback) { + // This will be handled by the compressor stream + callback(); + } +} + +module.exports = XzFileStream; diff --git a/lib/xz/index.js b/lib/xz/index.js new file mode 100644 index 0000000..62d4fcb --- /dev/null +++ b/lib/xz/index.js @@ -0,0 +1,55 @@ +'use strict'; + +const utils = require('../utils'); + +let XzFileStream; +let XzUncompressStream; + +function checkDependency() { + try { + require('lzma-native'); + return true; + } catch (err) { + return false; + } +} + +function throwIfNoDependency() { + if (!checkDependency()) { + throw new Error('lzma-native is required for xz compression/decompression. Please install it with: npm install lzma-native'); + } +} + +// Lazy load the implementation +function getImplementation() { + if (!XzFileStream) { + throwIfNoDependency(); + XzFileStream = require('./file_stream'); + XzUncompressStream = require('./uncompress_stream'); + } + return { XzFileStream, XzUncompressStream }; +} + +exports.FileStream = function(opts) { + const { XzFileStream } = getImplementation(); + return new XzFileStream(opts); +}; + +exports.UncompressStream = function(opts) { + const { XzUncompressStream } = getImplementation(); + return new XzUncompressStream(opts); +}; + +exports.compressFile = function(source, dest, opts) { + throwIfNoDependency(); + const { XzFileStream } = getImplementation(); + return utils.makeFileProcessFn(XzFileStream)(source, dest, opts); +}; + +exports.uncompress = function(source, dest, opts) { + throwIfNoDependency(); + const { XzUncompressStream } = getImplementation(); + return utils.makeFileProcessFn(XzUncompressStream)(source, dest, opts); +}; + +exports.decompress = exports.uncompress; diff --git a/lib/xz/uncompress_stream.js b/lib/xz/uncompress_stream.js new file mode 100644 index 0000000..8b1f48f --- /dev/null +++ b/lib/xz/uncompress_stream.js @@ -0,0 +1,53 @@ +'use strict'; + +const fs = require('fs'); +const lzma = require('lzma-native'); +const utils = require('../utils'); +const streamifier = require('streamifier'); +const { PassThrough } = require('stream'); + +class XzUncompressStream extends PassThrough { + constructor(opts) { + opts = opts || {}; + super(opts); + + const sourceType = utils.sourceType(opts.source); + // Set text mode to true to handle line endings correctly on Windows + const decompressor = lzma.createDecompressor({ + ...opts.lzma, + textMode: true, + }); + + decompressor.on('error', err => this.emit('error', err)); + decompressor.on('end', () => this.end()); + + // Handle single file decompression + if (sourceType === 'file') { + const stream = fs.createReadStream(opts.source, opts.fs); + stream.on('error', err => this.emit('error', err)); + stream.pipe(decompressor).pipe(this); + return; + } + + if (sourceType === 'buffer') { + const stream = streamifier.createReadStream(opts.source, opts.streamifier); + stream.on('error', err => this.emit('error', err)); + stream.pipe(decompressor).pipe(this); + return; + } + + if (sourceType === 'stream') { + opts.source.on('error', err => this.emit('error', err)); + opts.source.pipe(decompressor).pipe(this); + return; + } + + // For streaming input + this.on('pipe', srcStream => { + srcStream.unpipe(this); + srcStream.pipe(decompressor).pipe(this); + }); + } +} + +module.exports = XzUncompressStream; diff --git a/package.json b/package.json index 8e5a657..bf1f60d 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,9 @@ "@eggjs/yauzl": "^2.11.0", "yazl": "^2.4.2" }, + "optionalDependencies": { + "lzma-native": "^8.0.6" + }, "devDependencies": { "@types/mocha": "10", "@types/node": "20", diff --git a/test/fixtures/xx.log.xz b/test/fixtures/xx.log.xz new file mode 100644 index 0000000..1ae02ea Binary files /dev/null and b/test/fixtures/xx.log.xz differ diff --git a/test/xz/file_stream.test.js b/test/xz/file_stream.test.js new file mode 100644 index 0000000..7d04d17 --- /dev/null +++ b/test/xz/file_stream.test.js @@ -0,0 +1,110 @@ +const fs = require('fs'); +const path = require('path'); +const assert = require('assert'); +const compressing = require('../..'); + +describe('test/xz/file_stream.test.js', () => { + const sourceFile = path.join(__dirname, '../fixtures/xx.log'); + const xzFile = path.join(__dirname, '../fixtures/xx.log.xz'); + + it('should compress file to xz', done => { + const dest = path.join(__dirname, '../fixtures/xx.log.xz.tmp'); + compressing.xz.compressFile(sourceFile, dest) + .then(() => { + assert(fs.existsSync(dest)); + // 文件大小应该小于原始文件 + assert(fs.statSync(dest).size < fs.statSync(sourceFile).size); + fs.unlinkSync(dest); + done(); + }) + .catch(done); + }); + + it('should decompress xz file to log', done => { + const dest = path.join(__dirname, '../fixtures/xx.log.tmp'); + compressing.xz.uncompress(xzFile, dest) + .then(() => { + assert(fs.existsSync(dest)); + // 内容应该一致 + const raw = fs.readFileSync(sourceFile); + const out = fs.readFileSync(dest); + assert.equal(out.length, raw.length); + assert.deepEqual(out, raw); + fs.unlinkSync(dest); + done(); + }) + .catch(done); + }); + + it('should compress buffer to xz', done => { + const buf = fs.readFileSync(sourceFile); + const dest = path.join(__dirname, '../fixtures/xx.log.xz.tmp'); + compressing.xz.compressFile(buf, dest) + .then(() => { + assert(fs.existsSync(dest)); + fs.unlinkSync(dest); + done(); + }) + .catch(done); + }); + + it('should decompress xz buffer to log', done => { + const buf = fs.readFileSync(xzFile); + const dest = path.join(__dirname, '../fixtures/xx.log.tmp'); + compressing.xz.uncompress(buf, dest) + .then(() => { + assert(fs.existsSync(dest)); + const raw = fs.readFileSync(sourceFile); + const out = fs.readFileSync(dest); + assert.equal(out.length, raw.length); + assert.deepEqual(out, raw); + fs.unlinkSync(dest); + done(); + }) + .catch(done); + }); + + it('should compress/decompress utf-8 text to xz', async () => { + const buf = Buffer.from('你好\nhello xz\nWindows\r\n'); + const dest = path.join(__dirname, '../fixtures/xx.log.xz.utf8.tmp'); + await compressing.xz.compressFile(buf, dest); + assert(fs.existsSync(dest)); + + const dest2 = path.join(__dirname, '../fixtures/xx.log.utf8.tmp'); + const xzBuf = fs.readFileSync(dest); + await compressing.xz.uncompress(xzBuf, dest2); + const outBuf = fs.readFileSync(dest2); + assert.deepEqual(outBuf.toString(), buf.toString()); + + fs.unlinkSync(dest); + fs.unlinkSync(dest2); + }); + + it('should compress stream to xz', done => { + const src = fs.createReadStream(sourceFile); + const dest = path.join(__dirname, '../fixtures/xx.log.xz.tmp'); + compressing.xz.compressFile(src, dest) + .then(() => { + assert(fs.existsSync(dest)); + fs.unlinkSync(dest); + done(); + }) + .catch(done); + }); + + it('should decompress xz stream to log', done => { + const src = fs.createReadStream(xzFile); + const dest = path.join(__dirname, '../fixtures/xx.log.tmp'); + compressing.xz.uncompress(src, dest) + .then(() => { + assert(fs.existsSync(dest)); + const raw = fs.readFileSync(sourceFile); + const out = fs.readFileSync(dest); + assert.equal(out.length, raw.length); + assert.equal(out.toString(), raw.toString()); + fs.unlinkSync(dest); + done(); + }) + .catch(done); + }); +});