mhr3 / unzip-stream Goto Github PK

It's been a while since the last release. Do you mind bumping the version up and publishing a new release to the npm registry?

Here is the code:

'use strict';

const unzip = require('unzip-stream');

module.exports = (parts, moduleId) => {
  let noConfig = true;
  let idCheck = true;
  return new Promise(resolve => {
    parts
      .pipe(unzip.Parse())
      .on('entry', async entry => {
        if (entry.path !== 'config.json') {
          console.log(entry.path);
          entry.autodrain();
        } else {
          noConfig = false;
          entry.on('readable', () => {
            let chunk;
            while ((chunk = entry.read()) !== null) {
              const data = JSON.parse(chunk.toString());
              if (data.id !== moduleId) idCheck = false;
            }
          });
          entry.on('finish', () => {
            resolve({ noConfig, idCheck });
          });
        }
      })
      .on('end', () => {
        resolve({ noConfig, idCheck });
      });
  });
};

They will suddenly stopped when read after:

dist/
dist/app.a3c3f5c.css.map
dist/icon.png
dist/index.html
dist/.DS_Store
__MACOSX/
__MACOSX/dist/
__MACOSX/dist/._.DS_Store
dist/background.js
dist/manifest.c5c1480.js.map
dist/manifest.c5c1480.js
dist/vendor.f446120.js

Excuse me? What's wrong with my code? Or it is a bug?

dist.zip
dist 2.zip

This may be an issue with streams themselves but when trying to do the following:

read.pipe(unzip.Extract({path:sync9.projDir})).on('finish',function(){
	fs.rename(pLib.join(sync9.projDir,"workspace"),pLib.join(__dirname,"5700c92d-9db2-455b-83bd-53f0aaabf84d"),function(e){
		if(e) console.log(e)
		fs.rmdir(sync9.projDir,function(e){
			if(e) console.log(e)
			fs.rename(pLib.join(__dirname,"5700c92d-9db2-455b-83bd-53f0aaabf84d"),sync9.projDir,function(e){
				if(e) console.log(e)
				console.log("Local computer is now in sync with Cloud server.")
			})
		})
	})
})

I get an error:

{ Error: EPERM: operation not permitted, rename 'C:\Users\sancarn\workspace\LaunchMenu\c9 sy
nc\workspaces\sync9-testing\workspace' -> 'C:\Users\sancarn\workspace\LaunchMenu\c9 sync\570
0c92d-9db2-455b-83bd-53f0aaabf84d'
  errno: -4048,
  code: 'EPERM',
  syscall: 'rename',
  path: 'C:\\Users\\sancarn\\workspace\\LaunchMenu\\c9 sync\\workspaces\\sync9-testing\\work
space',
  dest: 'C:\\Users\\sancarn\\workspace\\LaunchMenu\\c9 sync\\5700c92d-9db2-455b-83bd-53f0aaa
bf84d' }

The reason why I get the error is because the files are still being accessed. My current work around is doing the following:

read.pipe(unzip.Extract({path:sync9.projDir})).on('finish',function(){
	setTimeout(function(){
		fs.rename(pLib.join(sync9.projDir,"workspace"),pLib.join(__dirname,"5700c92d-9db2-455b-83bd-53f0aaabf84d"),function(e){
			if(e) console.log(e)
			fs.rmdir(sync9.projDir,function(e){
				if(e) console.log(e)
				fs.rename(pLib.join(__dirname,"5700c92d-9db2-455b-83bd-53f0aaabf84d"),sync9.projDir,function(e){
					if(e) console.log(e)
					console.log("Local computer is now in sync with Cloud server.")
				})
			})
		})	
	},100)
})

Are there any better alternatives?

In the past (0.1.2) I used to have:

var unzipExtractor = unzip.Extract({
    path: workfolder
});

unzipExtractor.on('close', function () {
    // This line do fire
});

fs.createReadStream(myZipFile).pipe(unzipExtractor).on('entry', function (entry) {
    // Execute code for each entry - **THIS NEVER GETS EXECUTED - USED TO WORK IN 0.1.2**
});

This does not seem to work anymore. Any idea why not?

Hi, when packaging up unzip-stream into my bundle, I got an error that the octal literals are an "invalid number" and will only work in non-strict mode. I fixed this by changing them from 012 to 0o12 which is the es6 format for octal: https://github.com/lukehoban/es6features#binary-and-octal-literals

Acorn error: Invalid number (24946:40)
File: /Users/chris/Documents/WorkStuff/Deconet/Frontend/build/index.bundle.js

24944                  unixAttrs = this.parsedEntity.externalFileAttributes >>> 16;
24945                  var fileType = unixAttrs >>> 12;
24946                  isSymlink = (fileType & 012) === 012; // __S_IFLNK
24947              }
24948              if (this.options.debug) {
24949                  const debugObj = Object.assign({}, this.parsedEntity, {
24950                      path: path,

Pull request incoming :)

I found this // TODO in the source:

There are also many other uses of crc32 that could probably use an integrity check as well.

Hello, i would like to inquire why does this project depend on binary, a package that is 10 year old, instead of making use of the native Buffer and/or DataView methods?

The same could be said about mkdirp which as i understand could be easily achieved using the native fs?

Both CENTRAL_DIRECTORY_FILE_HEADER and CENTRAL_DIRECTORY_END_COMMENT are read but many properties are not passed to the entries. Things like the file comment are added to parsedEntity.extra but they are not exposed through UnzipStream.prototype._prepareOutStream. Using unzip.Parse({ debug: true }) shows them through stdout of course.

I was thinking to start a PR for this as it appears to be a few LOC to add. That being said, I'm not sure how this will impact the tests and will need to dig there.

Hi,
I use
fs.createReadStream('path/to/archive.zip').pipe(unzip.Extract({ path: 'output/path' }));
unzip file, How to on finish events?

0.2.2 introduces let outside of strict mode here (committed here), which breaks support for Node 4 and 5. I get errors like the following:

C:\projects\node-webrtc-7bnua\node_modules\unzip-stream\lib\unzip-stream.js:512
    let result = "";
    ^^^
SyntaxError: Block-scoped declarations (let, const, function, class) not yet supported outside strict mode
    at exports.runInThisContext (vm.js:53:16)
    at Module._compile (module.js:387:25)
    at Object.Module._extensions..js (module.js:422:10)
    at Module.load (module.js:357:32)
    at Function.Module._load (module.js:314:12)
    at Module.require (module.js:367:17)
    at require (internal/module.js:20:19)
    at Object.<anonymous> (C:\projects\node-webrtc-7bnua\node_modules\unzip-stream\lib\parser-stream.js:3:19)
    at Module._compile (module.js:413:34)
    at Object.Module._extensions..js (module.js:422:10)

Changes between 0.2.1 and 0.2.2

request(`https://bungie.net${manifest.mobileWorldContentPaths.en}`)
      .pipe(unzip.Extract({ path: `./manifests` }).on('close', () => {
        return fileName;
      }));

Expected result: The close event is emitted and fileName is returned.

Reality: No event is called. I've tried close, end, finish, and start and no event is ever emitted from Extract. I've tried putting it on the pipe and on Extract itself with the same result.

The PKZIP spec forbids null padding, inter alia, within local file record sequences:

• between a local file header record and its accompanying encryption header record (if present)
• between a local file header record and its accompanying file data record (if no encryption header record is present)
• between an encryption header record (if present) and its accompanying file data record
• between a file data record and its accompanying data descriptor record (if present)

Interestingly, it does not forbid null padding between such sequences.

I find myself sadly burdened with being required to deal with a lot of PKZIP files with null padding between local file record sequences on an ongoing basis for the foreseeable future. I have found that some PKZIP software (WinZip, Info-ZIP) handles these files easily, while other software (OS X, every PKZIP unzipper on npm) chokes and dies, including unzip-stream, which reports "Invalid signature in zip file".

It would make my life so, so much better if unzip-stream could support files with null padding between local file record sequences.

I'm getting the error "Stream finished in an invalid state, uncompression failed" on some ZIP files. Do you know what might be the cause? When it happens, is there a proper way to shut down the stream so that it no longer produces entries?

Hi, is there any way I could get the lastModified date or the entire file metadata for an entry?

Thanks

I try to pass decodeString property to Extract method like this:

fs.createReadStream(file.filepath)
.pipe(unzip.Extract({
decodeString: (buffer) => { return iconvLite.decode(buffer, 'UTF-8'); },
path: zipFilePath,
}))

But it doesnot work. After I review the source code, I found in the Extract function, when try to new UnzipStream, the param opts is forgotten to pass：
this.unzipStream = new UnzipStream(this.opts);

The full message (with debug logging) is:

Unexpected signature in zip file: 0x16d4b50 "PKm", skipped 1 bytes

This zip file is one that was created by the upload-artifact GitHub Action. The download-artifact action is unable to download the artifact because it's using unzip-stream which is unable to extract it.

I can provide the zip, but it's fairly large (938M) so I'm not sure if I should attach it here.

I wrote a function like this:

function extractZipEntriesFromBuffer (buffer, targets) {
  return new Promise((resolve, reject) => {
    const records = {}
    const unzipParser = unzip.Parse({debug: false})
    const stream = new PassThrough()
    stream.end(buffer)
    stream
      .pipe(unzipParser)
      .pipe(Transform({
        objectMode: true,
        transform: async function (entry, e, cb) {
          const {path, type, size} = entry
          if (targets.includes(path)) {
            records[path] = await streamToString(entry, cb)
            if (Object.keys(records).length === targets.length) {
              resolve(records)
            }
          } else {
            entry.autodrain()
            cb()
          }
        }
      }))
  })
}

It takes a buffer with the zip data. I am currently passing in a list of target files that I want extracted and when I have extracted those, I resolve the promise. I'd like to just resolve the promise when all entries have been received. I tried an on close event, but then it fires before the entry events even occur. The problem right now is if the caller happens to pass in a file entry that doesn't exist, then the promise doesn't resolve.

So the question is how do I know how many entries there are or when I finished receiving them all?

My zip has five files, but after I unziped i just get three of those.

Hi,
I'm having range error while trying to extract a zip file and it crashes my fastify app instead of raising an error. Any idea on what can be the reason and how to resolve it?
BTW, If I extract it and recompress it, the issue goes away.
Here is the code sniplet I am using to extract the files.

      fs.createReadStream(`${dir}/${filename}`)
        .pipe(unzip.Extract({ path: `${zipDir}` }))
        .on('close', () => {
          fastify.log.info(`Extracted zip ${zipDir}`);
          ...
        })
        .on('error', error => {
          reject(new InternalError(`Extracting zip ${filename}`, error));
        });

And here is the error:

RangeError [ERR_OUT_OF_RANGE]: The value of "offset" is out of range. It must be >= 0 and <= 5. Received 9
at boundsError (internal/buffer.js:77:9)
at Buffer.readUInt32LE (internal/buffer.js:211:5)
at UnzipStream._readExtraFields (/home/node/app/node_modules/unzip-stream/lib/unzip-stream.js:447:49)
at UnzipStream.processDataChunk (/home/node/app/node_modules/unzip-stream/lib/unzip-stream.js:213:30)
at UnzipStream._parseOrOutput (/home/node/app/node_modules/unzip-stream/lib/unzip-stream.js:651:28)
at done (/home/node/app/node_modules/unzip-stream/lib/unzip-stream.js:713:18)
at processTicksAndRejections (internal/process/task_queues.js:79:11)

I get error like:

Zip version 78.8 is not supported
at Immediate.<anonymous> (/myapp/node_modules/unzip-stream/lib/unzip-stream.js:355:33)
at processImmediate (node:internal/timers:464:21)

Entries are created in stream, can't find any way to subscribe on entry errors before UnzipStream.prototype._prepareOutStream function call. So every error emitted in this function throws exception.

When I got a bad zip to unzip, the app crashes, and there is no option to catch the error (except catching all global exceptions).

In the following app, I expect to get an error callback, but it get an exception: (events.js:291: "throw er; // Unhandled 'error' event") :

require("fs").createReadStream('bad.zip').pipe(require("unzip-stream") .Extract({ path: 'output' })) .on('error', (err)=>{ console.log(err); }).on("close", ()=>{ console.log("on closeed"); });
bad.zip

In the zip file, it has x permission, but without x permission after been extracted.

Is there a way to get the count of entries early in the stream processing?

Like could we emit a 'header' event or is there a field which is being set that I'm not seeing?

Possibly this information is not knowable without iterating the stream, then oh well but if it is available early on in stream processing I'd be really interested in getting that count somehow for progress reporting.

I understand the need for testing however testing the negative case (i.e. ensuring corrupt files are handled) seems problematic in this scenario. When including unzip-stream in my own app for example, virus scanners flag the invalid test data file as corrupt (as they should).

I have a zip file with multiple files. the problem here is that close fires before all Transforms are done.

What I want is to fire code in the unzipParser.on('close' after all entries was processed by transform: function(entry,e,cb) {

Also note that I do not know the number of files in the zip as this can be any number.

My Current Example (Not working as close is getting fired before all entries was read through the transform function):

var fs = require('fs');
var stream = require('stream');
var unzip = require('unzip-stream');

var workfolder = "/mytest";
var originalname = "mytest.zip";
var uType = "";

var unzipParser = unzip.Parse();
unzipParser.on('error', function (err) {
    console.error('Error unziping: ' + err);
});
unzipParser.on('close', function () {
    if (uType == "") {
        return res.status(200).send({
            success: false,
            message: 'ZIP file does not contain SHP or TAB files'
        });
    } else {
        res.status(200).json({
            success: true,
            message: 'Valid'
        });
    }
});    

fs.createReadStream(workfolder + "/" + originalname)
    .pipe(unzipParser)
    .pipe(stream.Transform({
        objectMode: true,
        transform: function(entry,e,cb) {
            var tFileName = entry.path;
            var tType = entry.type; // 'Directory' or 'File'
            var tSize = entry.size;
            if (tType === "File") {
                var tFileExt = tFileName.substr(tFileName.lastIndexOf('.') + 1);
                tFileExt = tFileExt.toUpperCase().trim();
                if (tFileExt == "SHP" || tFileExt == "TAB") {
                    uType = tFileExt;
                }
            }
            entry.pipe(fs.createWriteStream(workfolder + "/" + tFileName)).on('finish',cb);
        }
    }));

There is a vulnerability (CVE-2020-9493 ) in chainsaw package which is being used by the binary 0.3.0 package.

A deserialization flaw was found in Apache Chainsaw versions prior to 2.1.0 which could lead to malicious code execution

Even though that's a valid zip according to the spec.

For use in node / electron projects it would be good to adopt the polyfill from here https://nodejs.org/en/docs/guides/buffer-constructor-deprecation/#variant-2

I noticed that in the version check lines, an error is emitted for every zip file entry. I'm not intimately familiar with the zip file format but it looks like each file has a version in its header even though this appears to be a normalization problem with the format. In that case, would it suffice to emit an error on the unzip stream itself instead of each individual entry since, presumably, each entry has the same version. That is, instead of:

setImmediate(() => {
  entry.emit("error", new Error(message));
});

change the code to:

setImmediate(() => {
  self.emit("error", new Error(message));
});

Please let me know if I'm completely misinterpreting the intent of the code or zip file format. The same logic would apply for the encrypted flag. There don't appear to be any other events emitted for entry instances.

when I use this package to decompress a package whose size is almost 100M, I got a error: EMFILE, too many open files. here is part of code.

I want to read the file after unzip, but it failed, because the file was not written to disk now.
The follow is my code:
let fs = require('fs');
let path = require('path');
let unzip = require('unzip-stream');

let input = fs.createReadStream('test.zip');
let output = unzip.Extract({ path: path.join(__dirname, 'unzip') });

output.on('close', function(){
console.log('unzip close');
let data = fs.readFileSync(path.join(__dirname, './unzip/paper.json'), 'utf8');
});
output.on('error', function(err){
console.error(err.stack);
});

input.pipe(output);

I am using your package to download a release zip file from a GitHub project and extract it without writing the zip file to disk. The zip file on GitHub contains some shell scripts whose permissions are set to 750 in the zip file but following code is resulting in them having 644 permissions (the main issue for me is the lack of the executable bit being set).

const fetch = require('node-fetch');
...
const installFromZip = async (fileUrl, outputPath, proxyAgent) => {
  await fetch(fileUrl, getFetchOptions(proxyAgent)).then(res => {
    res.body.pipe(unzip.Extract({path: outputPath } ));
  });
};

Is there an option available to retain the file permissions when extracting? If not, can it be added?

How to unzip a zip file stream into memory? I would like to read the zip file as stream and pipe to memory i.e. using the .on('data' callback on the deflated buffer.