爬虫：爬取海词的翻译内容

在爬取海词的时候遇到了一个问题，在异步加载的时候，需要一个t值，但是这个t值是js加载出来的，直接拼接的，我们无法从网页中得到；

 当在搜索框输入单词的时候：你在干嘛

替换下图中的page的值就能达到翻页的目的：

那么当前的目的就是要能够找到这段js代码，同时获取对应输入的t的值，来重新组合url

真正的url只需要如下内容：

我提前把关键字和t都处理了，写成了字典的形式，

key：你在干嘛  ff[key]:WuqarCRs

{“你好”：“WuqarCRs”}  #提前处理成了这种，方便提取
url = "http://fuzz.dict.cn/dict/api.php?&action=fuzz&from=jsonp&q=" + key + "&t="+ ff[key]+"&page="

那么关键部分来了，我是如何获取T的呢。

大概思路，是在本地搭建一个服务器，然后输入每个词去访问这段js代码，然后返回给词的结果保存起来。

处理过程的代码：

第一步：先找到那段js代码，里面是如何把输入的文字转换成8位字符串的算法

第二步：先安装node.js 服务器，然后提取出来这段js代码，转换成node.js代码，如果不转换的话在浏览器里面直接访问时无法触发js加载的。

下面是node.js的代码，先执行node.js代码

var http = require(‘http‘);
var querystring = require(‘querystring‘);
var util = require(‘util‘);

http.createServer(function(req, res){
    var post = ‘‘;
    var mm = ‘‘

function dictCrypto(J) {
  function r(g, f) {
    var e, d, a, b, c;
    a = g & 2147483648;
    b = f & 2147483648;
    e = g & 1073741824;
    d = f & 1073741824;
    c = (g & 1073741823) + (f & 1073741823);
    if (e & d) {
      return c ^ 2147483648 ^ a ^ b
    }
    return e | d ? c & 1073741824 ? c ^ 3221225472 ^ a ^ b : c ^ 1073741824 ^ a ^ b : c ^ a ^ b
  }

  function I(g, f, e, d, a, b, c) {
    g = r(g, r(r(f & e | ~f & d, a), c));
    return r(g << b | g >>> 32 - b, f)
  }

  function s(g, f, e, d, a, b, c) {
    g = r(g, r(r(f & d | e & ~d, a), c));
    return r(g << b | g >>> 32 - b, f)
  }

  function w(g, f, e, d, a, b, c) {
    g = r(g, r(r(f ^ e ^ d, a), c));
    return r(g << b | g >>> 32 - b, f)
  }

  function v(g, f, e, d, a, b, c) {
    g = r(g, r(r(e ^ (f | ~d), a), c));
    return r(g << b | g >>> 32 - b, f)
  }

  function K(c) {
    for (var b = "++"; c > 0;) {
      var a = c % 64;
      b += a == 0 ? "+" : a == 1 ? "-" : a > 1 && a < 12 ? String.fromCharCode(a + 46) : a > 11 && a < 38 ? String.fromCharCode(a + 54) : String.fromCharCode(a + 59);
      c = (c - a) / 64
    }
    return b.substr(b.length - 2, 2)
  }

  function H(d) {
    var c = "",
      b = "",
      a;
    for (a = 0; a <= 3; a++) {
      b = d >>> a * 8 & 255;
      b = "0" + b.toString(16);
      c += b.substr(b.length - 2, 2)
    }
    return c
  }
  var x = [],G, L, q, p, F, E, D, C;
  J = function(d) {
      // var rrr = //;
      // d = d.replace(rrr,"");
      process.stdout.write(d +"***"+‘\n‘);
      process.stdout.write(typeof rrr);
    d = d.replace(/\r\n/g, "\n");

    for (var c = "",b = 0; b < d.length; b++) {
      var a = d.charCodeAt(b);
      if (a < 128) {
        c += String.fromCharCode(a)
      } else {
        if (a > 127 && a < 2048) {
          c += String.fromCharCode(a >> 6 | 192)
        } else {
          c += String.fromCharCode(a >> 12 | 224);
          c += String.fromCharCode(a >> 6 & 63 | 128)
        }
        c += String.fromCharCode(a & 63 | 128)
      }
    }
    c += String.fromCharCode(80, 97, 83, 115);
    if (global.dict_pagetoken) {
      c += global.dict_pagetoken
    }
    return c
  }(J);
  x = function(g) {
    var f, e = g.length;
    f = e + 8;
    for (var d = ((f - f % 64) / 64 + 1) * 16, a = Array(d - 1), b = 0, c = 0; c < e;) {
      f = (c - c % 4) / 4;
      b = c % 4 * 8;
      a[f] |= g.charCodeAt(c) << b;
      c++
    }
    a[(c - c % 4) / 4] |= 128 << c % 4 * 8;
    a[d - 2] = e << 3;
    a[d - 1] = e >>> 29;
    return a
  }(J);
  F = 1732584193;
  E = 4023233417;
  D = 2562383102;
  C = 271733878;
  for (J = 0; J < x.length; J += 16) {
    G = F;
    L = E;
    q = D;
    p = C;
    F = I(F, E, D, C, x[J + 0], 7, 3614090360);
    C = I(C, F, E, D, x[J + 1], 12, 3905402710);
    D = I(D, C, F, E, x[J + 2], 17, 606105819);
    E = I(E, D, C, F, x[J + 3], 22, 3250441966);
    F = I(F, E, D, C, x[J + 4], 7, 4118548399);
    C = I(C, F, E, D, x[J + 5], 12, 1200080426);
    D = I(D, C, F, E, x[J + 6], 17, 2821735955);
    E = I(E, D, C, F, x[J + 7], 22, 4249261313);
    F = I(F, E, D, C, x[J + 8], 7, 1770035416);
    C = I(C, F, E, D, x[J + 9], 12, 2336552879);
    D = I(D, C, F, E, x[J + 10], 17, 4294925233);
    E = I(E, D, C, F, x[J + 11], 22, 2304563134);
    F = I(F, E, D, C, x[J + 12], 7, 1804603682);
    C = I(C, F, E, D, x[J + 13], 12, 4254626195);
    D = I(D, C, F, E, x[J + 14], 17, 2792965006);
    E = I(E, D, C, F, x[J + 15], 22, 1236535329);
    F = s(F, E, D, C, x[J + 1], 5, 4129170786);
    C = s(C, F, E, D, x[J + 6], 9, 3225465664);
    D = s(D, C, F, E, x[J + 11], 14, 643717713);
    E = s(E, D, C, F, x[J + 0], 20, 3921069994);
    F = s(F, E, D, C, x[J + 5], 5, 3593408605);
    C = s(C, F, E, D, x[J + 10], 9, 38016083);
    D = s(D, C, F, E, x[J + 15], 14, 3634488961);
    E = s(E, D, C, F, x[J + 4], 20, 3889429448);
    F = s(F, E, D, C, x[J + 9], 5, 568446438);
    C = s(C, F, E, D, x[J + 14], 9, 3275163606);
    D = s(D, C, F, E, x[J + 3], 14, 4107603335);
    E = s(E, D, C, F, x[J + 8], 20, 1163531501);
    F = s(F, E, D, C, x[J + 13], 5, 2850285829);
    C = s(C, F, E, D, x[J + 2], 9, 4243563512);
    D = s(D, C, F, E, x[J + 7], 14, 1735328473);
    E = s(E, D, C, F, x[J + 12], 20, 2368359562);
    F = w(F, E, D, C, x[J + 5], 4, 4294588738);
    C = w(C, F, E, D, x[J + 8], 11, 2272392833);
    D = w(D, C, F, E, x[J + 11], 16, 1839030562);
    E = w(E, D, C, F, x[J + 14], 23, 4259657740);
    F = w(F, E, D, C, x[J + 1], 4, 2763975236);
    C = w(C, F, E, D, x[J + 4], 11, 1272893353);
    D = w(D, C, F, E, x[J + 7], 16, 4139469664);
    E = w(E, D, C, F, x[J + 10], 23, 3200236656);
    F = w(F, E, D, C, x[J + 13], 4, 681279174);
    C = w(C, F, E, D, x[J + 0], 11, 3936430074);
    D = w(D, C, F, E, x[J + 3], 16, 3572445317);
    E = w(E, D, C, F, x[J + 6], 23, 76029189);
    F = w(F, E, D, C, x[J + 9], 4, 3654602809);
    C = w(C, F, E, D, x[J + 12], 11, 3873151461);
    D = w(D, C, F, E, x[J + 15], 16, 530742520);
    E = w(E, D, C, F, x[J + 2], 23, 3299628645);
    F = v(F, E, D, C, x[J + 0], 6, 4096336452);
    C = v(C, F, E, D, x[J + 7], 10, 1126891415);
    D = v(D, C, F, E, x[J + 14], 15, 2878612391);
    E = v(E, D, C, F, x[J + 5], 21, 4237533241);
    F = v(F, E, D, C, x[J + 12], 6, 1700485571);
    C = v(C, F, E, D, x[J + 3], 10, 2399980690);
    D = v(D, C, F, E, x[J + 10], 15, 4293915773);
    E = v(E, D, C, F, x[J + 1], 21, 2240044497);
    F = v(F, E, D, C, x[J + 8], 6, 1873313359);
    C = v(C, F, E, D, x[J + 15], 10, 4264355552);
    D = v(D, C, F, E, x[J + 6], 15, 2734768916);
    E = v(E, D, C, F, x[J + 13], 21, 1309151649);
    F = v(F, E, D, C, x[J + 4], 6, 4149444226);
    C = v(C, F, E, D, x[J + 11], 10, 3174756917);
    D = v(D, C, F, E, x[J + 2], 15, 718787259);
    E = v(E, D, C, F, x[J + 9], 21, 3951481745);
    F = r(F, G);
    E = r(E, L);
    D = r(D, q);
    C = r(C, p)
  }
  return function(d) {
      var c = parseInt("0x" + d.substr(0, 3), 16),
        b = parseInt("0x" + d.substr(3, 3), 16),
        a = parseInt("0x" + d.substr(6, 3), 16);
      d = parseInt("0x" + d.substr(9, 3), 16);
      return K(c) + K(b) + K(a) + K(d);
      console.log(K(c) + K(b) + K(a) + K(d))
    }
    (H(F).substr(0, 4) + H(E).substr(0, 4) + H(D).substr(0, 4))
}
  //传过来的时候，chunk = “你好”
    req.on(‘data‘, function(chunk){    #添加post请求
        process.stdout.write(chunk+‘\n‘);
        // // process.stdout.write(hh + ‘\n‘);
        // var hhh = "你好"
        rrr = chunk.toString()
        process.stdout.write(typeof rrr  + ‘\n‘);

        process.stdout.write(rrr+‘\n‘);
        mm = dictCrypto(rrr);
        post +=mm;
    });

    req.on(‘end‘, function(){
        post = querystring.parse(post);
        res.end(util.inspect(post));
    });
}).listen(8888);

console.log(‘Server running at http://127.0.0.1:8888/‘);

第三步：正常的python代码，去访问本地的服务器，直接把转换完的数据存储到本地

#! /usr/bin/env python
#coding: utf-8
import re
import os
import requests
import sys
import json
reload(sys)
sys.setdefaultencoding(‘utf-8‘)
path = "D:\\106_data\\juhai_data\\"
ff = open(path + "answer_1.txt",‘a‘)
f = open("data_1.dict")   #这个是你的词典，按照行来访问词典
tt = {}
i = 1
j = 1

s = requests.session()
s.keep_alive = False

while 1:
    word = f.readline()
    if not word:
        ans = json.dumps(tt)
        ff.write(ans)
        break
    print word,
    if (i%100000 == 0):#一万个词存储一次，存的格式为字典
        j = j + 1
        ans = json.dumps(tt)
        ff.write(ans)
        ff.close()
        ff = open(path + "answer_" +str(j) + ".txt",‘a‘)
        tt = {}
    word = word.strip(‘\n‘)
    html = requests.post("http://127.0.0.1:8888/",data =word,headers={‘Connection‘:‘close‘})
    print html.text
    xx = re.search("{ (.*?): ‘‘ }",html.text,re.S)#用到了正则去提取内容
    try:
        xx = xx.group(1)
        xx = xx.strip("‘")
    except:
        continue
    tt[word] = xx
    print xx
    i = i+1
    s = requests.session()
    s.keep_alive = False
ff.close()
f.close()