30秒后,Casperjs文件下载时间[英] CasperJS File Download Times Out After 30 Seconds

问题描述

我正在使用Casperjs下载15 MB文件.在浏览器中,下载大约需要3分钟才能完成.使用Casper,恰好在30秒后返回相同URL的.download函数,并且写入磁盘的文件为0字节.我尝试设置这样的时间更长的超时:

var casper = require("casper").create({
    pageSettings: {
        webSecurityEnabled: false
    },
    waitTimeout: 500000,
    stepTimeout: 500000
});

,但它们没有效果.这是我的下载功能:

casper.on('resource.received', function (resource) {
    var url, file;
    if ((resource.url.indexOf("myDownloadUniqueString=") !== -1) ) {
        this.echo(resource.url);  // the echo'ed url can be downloaded in a web browser
        url = resource.url;
        file = "downloaded_file.wav";  // this will be 0 bytes
        try {
            var fs = require("fs"); // phantom js file system (not node)
            casper.download(resource.url, file);
        } catch (e) {
            this.echo(e);  // no error is thrown
        }
    }
});

有什么想法吗?也许是phantomjs fs方法的问题,但是该文档不完整. ..

推荐答案

i通过在评估函数中运行异步XMLHTTPRequest(XHR),通过运行异步XMLHTTPREQUEST(XHR)来解决此问题(对于Excel .xls二进制下载> 30s,6mb大约),然后将结果写入全局(window)变量,等待设置这个全局.

在XHR对象上,您可以在下面的示例中设置自定义超时,15 * 60 * 1000 = 15分钟.

需要使用以ASCII/BASE64为单位的二进制下载来进行护理,然后将它们解码以编写二进制文件.可以对此进行调整/简化以供文本下载.

var fs = require('fs');
var casper = require('casper').create({
    //options here
});

var xhr = this.evaluate(function(url){

    var xhr = new XMLHttpRequest();
    xhr.timeout = 15 * 60 * 1000;
    xhr.overrideMimeType("text/plain; charset=x-user-defined");
    xhr.open("GET", url);  // synchronous request banned, so use waitfor to wait on a global variable

    xhr.onreadystatechange = function () {
        if (xhr.readyState == 4) {
            //if (xhr.status == 200) { 
            window.xhrstatus = xhr.status;    //write to a global (window) variable
            window.xhrResponseText = __utils__.encode(xhr.responseText);   //base64 encode using casper functions (btoa fails)
            //}
        }
    };                        

    xhr.send(null);
    return true;
},'http://example.com/download.xls');


casper.waitFor(function() { 
    return this.getGlobal('xhrstatus') != undefined; 
}, function() { 
    this.echo('XHR status: ' + this.getGlobal('xhrstatus')); 
    this.echo('Saving report...'); 
    //http://phantomjs.org/api/fs/method/is-writable.html to check if file writable first
    //decode using casper clientutil function and then write binary file
    fs.write('saveFileName.xls', decode(this.getGlobal('xhrResponseText')), 'wb');
},null,15*60*1000);

casper.js客户端库中的编码/解码函数看起来像这样.这些似乎在JavaScript的ATOB()和BTOA()不作用.

/* 
 * encode / decode function from casper.js clientutils 
 * https://github.com/casperjs/casperjs/blob/master/modules/clientutils.js
 * Included here for reference - you could just reference the file in your code
*/
var BASE64_ENCODE_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
var BASE64_DECODE_CHARS = [
    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63,
    52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1,
    -1,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14,
    15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1,
    -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
    41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1
];

/**
 * Decodes a base64 encoded string. Succeeds where window.atob() fails.
 *
 * @param  String  str  The base64 encoded contents
 * @return string
 */
var decode = function decode(str) {
    /*eslint max-statements:0, complexity:0 */
    var c1, c2, c3, c4, i = 0, len = str.length, out = "";
    while (i < len) {
        do {
            c1 = BASE64_DECODE_CHARS[str.charCodeAt(i++) & 0xff];
        } while (i < len && c1 === -1);
        if (c1 === -1) {
            break;
        }
        do {
            c2 = BASE64_DECODE_CHARS[str.charCodeAt(i++) & 0xff];
        } while (i < len && c2 === -1);
        if (c2 === -1) {
            break;
        }
        out += String.fromCharCode(c1 << 2 | (c2 & 0x30) >> 4);
        do {
            c3 = str.charCodeAt(i++) & 0xff;
            if (c3 === 61) {
                return out;
            }
            c3 = BASE64_DECODE_CHARS[c3];
        } while (i < len && c3 === -1);
        if (c3 === -1) {
            break;
        }
        out += String.fromCharCode((c2 & 0XF) << 4 | (c3 & 0x3C) >> 2);
        do {
            c4 = str.charCodeAt(i++) & 0xff;
            if (c4 === 61) {
                return out;
            }
            c4 = BASE64_DECODE_CHARS[c4];
        } while (i < len && c4 === -1);
        if (c4 === -1) {
            break;
        }
        out += String.fromCharCode((c3 & 0x03) << 6 | c4);
    }
    return out;
};    

/**
 * Base64 encodes a string, even binary ones. Succeeds where
 * window.btoa() fails.
 *
 * @param  String  str  The string content to encode
 * @return string
 */
var encode = function encode(str) {
    /*eslint max-statements:0 */
    var out = "", i = 0, len = str.length, c1, c2, c3;
    while (i < len) {
        c1 = str.charCodeAt(i++) & 0xff;
        if (i === len) {
            out += BASE64_ENCODE_CHARS.charAt(c1 >> 2);
            out += BASE64_ENCODE_CHARS.charAt((c1 & 0x3) << 4);
            out += "==";
            break;
        }
        c2 = str.charCodeAt(i++);
        if (i === len) {
            out += BASE64_ENCODE_CHARS.charAt(c1 >> 2);
            out += BASE64_ENCODE_CHARS.charAt((c1 & 0x3) << 4 | (c2 & 0xF0) >> 4);
            out += BASE64_ENCODE_CHARS.charAt((c2 & 0xF) << 2);
            out += "=";
            break;
        }
        c3 = str.charCodeAt(i++);
        out += BASE64_ENCODE_CHARS.charAt(c1 >> 2);
        out += BASE64_ENCODE_CHARS.charAt((c1 & 0x3) << 4 | (c2 & 0xF0) >> 4);
        out += BASE64_ENCODE_CHARS.charAt((c2 & 0xF) << 2 | (c3 & 0xC0) >> 6);
        out += BASE64_ENCODE_CHARS.charAt(c3 & 0x3F);
    }
    return out;
};

其他推荐答案

添加resourceTimeout:

如何
pageSettings: {
  webSecurityEnabled: false,
  resourceTimeout: 240000 //240s
},

这个答案表示它已添加到Phantomjs 1.9中,但尚未记录.

本文地址:https://www.itbaoku.cn/post/1739876.html