XiaoMo/anyproxy
1
0
Fork 0
mirror of https://github.com/alibaba/anyproxy.git synced 2025-05-08 22:08:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

move response data from memory to local cache file

Browse Source
This commit is contained in:
OttoMao 2015-10-24 23:37:43 +08:00
parent 31daeebdd5
commit cfc817cb1f
13 changed files with 248 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
lib/recorder.js
View File

@ -2,15 +2,18 @@
var zlib = require('zlib'),
Datastore = require('nedb'),
util = require("util"),
path = require("path"),
fs = require("fs"),
events = require('events'),
iconv = require('iconv-lite'),
proxyUtil = require("./util"),
logUtil = require("./log");
//option.filename
function Recorder(option){
var self = this,
id = 1,
id = 1,
cachePath = proxyUtil.generateCacheDir(),
db;
option = option || {};
@ -87,31 +90,40 @@ function Recorder(option){
};
//update recordBody if exits
//TODO : trigger update callback
var BODY_FILE_PRFIX = "res_body_";
self.updateRecordBody =function(id,info){
if(id == -1) return;
if(!id || !info.resBody) return;
//add to body map
//ignore image data
if(/image/.test(info.resHeader['content-type'])){
self.recordBodyMap[id] = "(image)";
}else{
self.recordBodyMap[id] = info.resBody;
}
var bodyFile = path.join(cachePath,BODY_FILE_PRFIX + id);
fs.writeFile(bodyFile, info.resBody);
};
self.getBody = function(id){
self.getBody = function(id,cb){
if(id < 0){
return "";
cb && cb("");
}
return self.recordBodyMap[id] || "";
var bodyFile = path.join(cachePath,BODY_FILE_PRFIX + id);
fs.access(bodyFile, fs.F_OK | fs.R_OK ,function(err){
if(err){
cb && cb(err);
}else{
fs.readFile(bodyFile,cb);
}
});
};
self.getBodyUTF8 = function(id,cb){
var bodyContent = self.getBody(id),
result = "";
self.getDecodedBody = function(id,cb){
var result = {
type : "unknown",
mime : "",
content : ""
};
GLOBAL.recorder.getSingleRecord(id,function(err,doc){
//check whether this record exists
if(!doc || !doc[0]){
@ -119,24 +131,39 @@ function Recorder(option){
return;
}
if(!bodyContent){
cb(null,result);
}else{
var record = doc[0],
resHeader = record['resHeader'] || {};
try{
var charsetMatch = JSON.stringify(resHeader).match(/charset="?([a-zA-Z0-9\-]+)"?/);
if(charsetMatch && charsetMatch.length > 1){
var currentCharset = charsetMatch[1].toLowerCase();
if(currentCharset != "utf-8" && iconv.encodingExists(currentCharset)){
bodyContent = iconv.decode(bodyContent, currentCharset);
self.getBody(id,function(err,bodyContent){
if(err){
cb(err);
}else if(!bodyContent){
cb(null,result);
}else{
var record = doc[0],
resHeader = record['resHeader'] || {};
try{
var headerStr = JSON.stringify(resHeader),
charsetMatch = headerStr.match(/charset="?([a-zA-Z0-9\-]+)"?/),
imageMatch = resHeader && resHeader["content-type"];
if(charsetMatch && charsetMatch.length){
var currentCharset = charsetMatch[1].toLowerCase();
if(currentCharset != "utf-8" && iconv.encodingExists(currentCharset)){
bodyContent = iconv.decode(bodyContent, currentCharset);
}
result.type = "text";
result.content = bodyContent.toString();
}else if(imageMatch){
result.type = "image";
result.mime = imageMatch;
result.content = bodyContent;
}
}
}catch(e){}
}catch(e){}
cb(null,bodyContent.toString());
}
cb(null,result);
}
});
});
};

4
lib/requestHandler.js
View File

@ -24,6 +24,7 @@ function userRequestHandler(req,userRes){
in http server : http://www.example.com/a/b/c
in https server : /a/b/c
*/
var host = req.headers.host,
protocol = (!!req.connection.encrypted && !/^http:/.test(req.url)) ? "https" : "http",
fullUrl = protocol === "http" ? req.url : (protocol + '://' + host + req.url),
@ -33,6 +34,9 @@ function userRequestHandler(req,userRes){
resourceInfoId = -1,
reqData;
// console.log(req.url);
// console.log(path);
//record
resourceInfo = {
host : host,

25
lib/util.js
View File

@ -1,5 +1,7 @@
var fs = require("fs"),
path = require("path");
path = require("path"),
exec = require('child_process').exec;
// {"Content-Encoding":"gzip"} --> {"content-encoding":"gzip"}
module.exports.lower_keys = function(obj){
@ -41,6 +43,27 @@ module.exports.getAnyProxyHome = function(){
return home;
}
var CACHE_DIR_PREFIX = "cache_r";
module.exports.generateCacheDir = function(){
var rand = Math.floor(Math.random() * 1000000),
cachePath = path.join(util.getAnyProxyHome(),"./" + CACHE_DIR_PREFIX + rand);
fs.mkdirSync(cachePath,0777);
return cachePath;
}
module.exports.clearCacheDir = function(cb){
var home = util.getAnyProxyHome(),
isWin = /^win/.test(process.platform);
var dirNameWildCard = CACHE_DIR_PREFIX + "*";
if(isWin){
exec("for /D %f in (" + dirNameWildCard + ") do rmdir %f /s /q",{cwd : home},cb);
}else{
exec("rm -rf " + dirNameWildCard + "",{cwd : home},cb);
}
}
module.exports.simpleRender = function(str, object, regexp){
return String(str).replace(regexp || (/\{\{([^{}]+)\}\}/g), function(match, name){
if (match.charAt(0) == '\\') return match.slice(1);

30
lib/webInterface.js
View File

@ -48,6 +48,36 @@ function webInterface(config){
});
});
app.get("/fetchBody",function(req,res){
var query = req.query;
if(query && query.id){
GLOBAL.recorder.getDecodedBody(query.id, function(err, result){
if(err || !result || !result.content){
res.json({});
}else if(result.type && result.type == "image" && result.mime){
if(query.raw){
//TODO : cache query result
res.type(result.mime).end(result.content);
}else{
res.json({
id : query.id,
type : result.type,
ref : "/fetchBody?id=" + query.id + "&raw=true"
});
}
}else{
res.json({
id : query.id,
type : result.type,
content : result.content
});
}
});
}else{
res.end({});
}
});
app.get("/fetchCrtFile",function(req,res){
if(crtFilePath){
res.setHeader("Content-Type","application/x-x509-ca-cert");

6
lib/wsServer.js
View File

@ -23,8 +23,8 @@ function resToMsg(msg,cb){
}
if(jsonData.type == "reqBody" && jsonData.id){
result.type ="body";
GLOBAL.recorder.getBodyUTF8(jsonData.id, function(err, data){
result.type = "body";
GLOBAL.recorder.getBody(jsonData.id, function(err, data){
if(err){
result.content = {
id : null,
@ -34,7 +34,7 @@ function resToMsg(msg,cb){
}else{
result.content = {
id : jsonData.id,
body : data
body : data.toString()
};
}
cb && cb(result);

2
package.json
View File

@ -25,7 +25,7 @@
"ws": "^0.4.32"
},
"devDependencies": {
"proxy-eval": ">=1.1.1"
"proxy-eval": ">=1.1.2"
},
"scripts": {
"test": "sh test.sh"

18
proxy.js
View File

@ -69,11 +69,7 @@ function proxyServer(option){
logUtil.setPrintStatus(false);
}
if(option.dbFile){
GLOBAL.recorder = new Recorder({filename: option.dbFile});
}else{
GLOBAL.recorder = new Recorder();
}
if(!!option.interceptHttps){
default_rule.setInterceptFlag(true);
@ -95,6 +91,18 @@ function proxyServer(option){
async.series(
[
//clear cache dir, prepare recorder
function(callback){
util.clearCacheDir(function(){
if(option.dbFile){
GLOBAL.recorder = new Recorder({filename: option.dbFile});
}else{
GLOBAL.recorder = new Recorder();
}
callback();
});
},
//creat proxy server
function(callback){
if(proxyType == T_TYPE_HTTPS){

37
test/large_post.js Normal file
View File

@ -0,0 +1,37 @@
var proxy = require("../proxy.js"),
proxyTester = require("proxy-eval"),
WebSocket = require("ws"),
Buffer = require("buffer").Buffer,
express = require("express");
var app = express()
app.post('/', function (req, res) {
var bigBody = new Buffer(1024 * 1024 * 10);
res.send( bigBody ); //10 mb
});
app.listen(3000);
function test(){
//test the basic availibility of proxy server
setTimeout(function(){
var testParam = {
proxy : 'http://127.0.0.1:8001/',
reqTimeout : 4500,
httpGetUrl : "",
httpPostUrl : "http://127.0.0.1:3000/",
httpPostBody : "123",
httpsGetUrl : "",
httpsPostUrl : "",
httpsPostBody : ""
};
proxyTester.test(testParam ,function(results){
process.exit();
});
},1000);
};
setTimeout(function(){
test();
},3000);

8
test/test.js
View File

@ -44,7 +44,13 @@ exports.avalibility = function(test){
//test the basic availibility of proxy server
setTimeout(function(){
proxyTester.test({proxy : 'http://127.0.0.1:8995',reqTimeout:4500} ,function(results){
var testParam = {
proxy : 'http://127.0.0.1:8995',
reqTimeout : 4500,
httpsPostUrl : "http://www.sample.com/"
httpsPostBody : "123"
};
proxyTester.test(testParam ,function(results){
var successCount = 0;
results.map(function(item){
item.success && ++successCount;

30
web/build/detailPanel.js
View File

@ -12,11 +12,23 @@ function init(React){
id = self.props.data.id;
if(!id) return;
ws.reqBody(id,function(content){
if(content.id == self.props.data.id){
self.setState({
body : content
});
jQuery.get("/fetchBody?id=" + id ,function(resObj){
if(resObj && resObj.id){
if(resObj.type && resObj.type == "image" && resObj.ref){
self.setState({
body : {
img : resObj.ref,
id : resObj.id
}
});
}else if(resObj.content){
self.setState({
body : {
body : resObj.content,
id : resObj.id
}
});
}
}
});
},
@ -55,9 +67,13 @@ function init(React){
);
if(this.props.data.statusCode){
if(this.state.body.id == this.props.data.id){
bodyContent = (React.createElement("pre", {className: "resBodyContent"}, this.state.body.body));
if(this.state.body.img){
var imgEl = { __html : '<img src="'+ this.state.body.img +'" />'};
bodyContent = (React.createElement("div", {dangerouslySetInnerHTML: imgEl}));
}else{
bodyContent = (React.createElement("pre", {className: "resBodyContent"}, this.state.body.body));
}
}else{
bodyContent = null;
this.loadBody();

5
web/css/page.css
View File

@ -186,6 +186,11 @@ body, html {
word-wrap:break-word;
}
.resBody .resBodyContent img{
max-width: 500px;
max-height: 500px;
}
.subTitle{
padding-left: 6px;
border-left: 3px solid #1FA2D6;

30
web/page.js
View File

@ -20342,11 +20342,23 @@
id = self.props.data.id;
if(!id) return;
ws.reqBody(id,function(content){
if(content.id == self.props.data.id){
self.setState({
body : content
});
jQuery.get("/fetchBody?id=" + id ,function(resObj){
if(resObj && resObj.id){
if(resObj.type && resObj.type == "image" && resObj.ref){
self.setState({
body : {
img : resObj.ref,
id : resObj.id
}
});
}else if(resObj.content){
self.setState({
body : {
body : resObj.content,
id : resObj.id
}
});
}
}
});
},
@ -20385,9 +20397,13 @@
);
if(this.props.data.statusCode){
if(this.state.body.id == this.props.data.id){
bodyContent = (React.createElement("pre", {className: "resBodyContent"}, this.state.body.body));
if(this.state.body.img){
var imgEl = { __html : '<img src="'+ this.state.body.img +'" />'};
bodyContent = (React.createElement("div", {dangerouslySetInnerHTML: imgEl}));
}else{
bodyContent = (React.createElement("pre", {className: "resBodyContent"}, this.state.body.body));
}
}else{
bodyContent = null;
this.loadBody();

30
web/src/detailPanel.js
View File

@ -12,11 +12,23 @@ function init(React){
id = self.props.data.id;
if(!id) return;
ws.reqBody(id,function(content){
if(content.id == self.props.data.id){
self.setState({
body : content
});
jQuery.get("/fetchBody?id=" + id ,function(resObj){
if(resObj && resObj.id){
if(resObj.type && resObj.type == "image" && resObj.ref){
self.setState({
body : {
img : resObj.ref,
id : resObj.id
}
});
}else if(resObj.content){
self.setState({
body : {
body : resObj.content,
id : resObj.id
}
});
}
}
});
},
@ -55,9 +67,13 @@ function init(React){
);
if(this.props.data.statusCode){
if(this.state.body.id == this.props.data.id){
bodyContent = (<pre className="resBodyContent">{this.state.body.body}</pre>);
if(this.state.body.img){
var imgEl = { __html : '<img src="'+ this.state.body.img +'" />'};
bodyContent = (<div dangerouslySetInnerHTML={imgEl}></div>);
}else{
bodyContent = (<pre className="resBodyContent">{this.state.body.body}</pre>);
}
}else{
bodyContent = null;
this.loadBody();