scrapers.js 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381
  1. // Generated by IcedCoffeeScript 1.7.1-c
  2. (function() {
  3. var BAD_X, Base, CHECK, Coinbase, Dns, E, GenericWebSite, Github, HackerNews, Proof, Reddit, SocialNetwork, Twitter, alloc_stub, cheerio, colors, env, iced, log, my_request, proofs, proxyca, request, root_certs, semver, tor, __iced_k, __iced_k_noop, _certs, _ref,
  4. __hasProp = {}.hasOwnProperty,
  5. __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
  6. iced = require('iced-runtime').iced;
  7. __iced_k = __iced_k_noop = function() {};
  8. log = require('./log');
  9. request = require('request');
  10. cheerio = require('cheerio');
  11. env = require('./env').env;
  12. proofs = require('keybase-proofs');
  13. E = require('./err').E;
  14. _ref = require('./display'), CHECK = _ref.CHECK, BAD_X = _ref.BAD_X;
  15. colors = require('colors');
  16. proxyca = require('./proxyca');
  17. root_certs = require('../json/node_root_certs.json');
  18. semver = require('semver');
  19. Proof = require('libkeybase').assertion.Proof;
  20. tor = require('./tor');
  21. _certs = null;
  22. my_request = function(opts, cb) {
  23. var k, v;
  24. if (semver.lt(process.version, "0.10.26")) {
  25. if (_certs == null) {
  26. _certs = (function() {
  27. var _results;
  28. _results = [];
  29. for (k in root_certs) {
  30. v = root_certs[k];
  31. _results.push(v);
  32. }
  33. return _results;
  34. })();
  35. }
  36. if (opts.ca == null) {
  37. opts.ca = _certs;
  38. }
  39. }
  40. tor.agent(opts);
  41. return request(opts, cb);
  42. };
  43. Base = (function() {
  44. function Base() {
  45. this.make_scraper();
  46. }
  47. Base.prototype.make_scraper = function() {
  48. var klass, _ref1;
  49. klass = this.get_scraper_klass();
  50. return this._scraper = new klass({
  51. libs: {
  52. cheerio: cheerio,
  53. request: my_request,
  54. log: log
  55. },
  56. log_level: 'debug',
  57. proxy: env().get_proxy(),
  58. ca: (_ref1 = proxyca.get()) != null ? _ref1.data() : void 0
  59. });
  60. };
  61. Base.prototype.single_occupancy = function() {
  62. return false;
  63. };
  64. Base.prototype.scraper = function() {
  65. return this._scraper;
  66. };
  67. Base.prototype.get_sub_id = function() {
  68. return null;
  69. };
  70. Base.prototype.validate = function(arg, cb) {
  71. var err, rc, ___iced_passed_deferral, __iced_deferrals, __iced_k, _ref1;
  72. __iced_k = __iced_k_noop;
  73. ___iced_passed_deferral = iced.findDeferral(arguments);
  74. if (tor.enabled()) {
  75. _ref1 = this._scraper.get_tor_error(arg), err = _ref1[0], rc = _ref1[1];
  76. }
  77. (function(_this) {
  78. return (function(__iced_k) {
  79. if (err == null) {
  80. (function(__iced_k) {
  81. __iced_deferrals = new iced.Deferrals(__iced_k, {
  82. parent: ___iced_passed_deferral,
  83. filename: "/home/max/src/keybase/node-client/src/scrapers.iced",
  84. funcname: "Base.validate"
  85. });
  86. _this._scraper.validate(arg, __iced_deferrals.defer({
  87. assign_fn: (function() {
  88. return function() {
  89. err = arguments[0];
  90. return rc = arguments[1];
  91. };
  92. })(),
  93. lineno: 60
  94. }));
  95. __iced_deferrals._fulfill();
  96. })(__iced_k);
  97. } else {
  98. return __iced_k();
  99. }
  100. });
  101. })(this)((function(_this) {
  102. return function() {
  103. return cb(err, rc);
  104. };
  105. })(this));
  106. };
  107. return Base;
  108. })();
  109. SocialNetwork = (function(_super) {
  110. __extends(SocialNetwork, _super);
  111. function SocialNetwork() {
  112. return SocialNetwork.__super__.constructor.apply(this, arguments);
  113. }
  114. SocialNetwork.prototype.format_msg = function(_arg) {
  115. var arg, ok;
  116. arg = _arg.arg, ok = _arg.ok;
  117. return [(ok ? CHECK : BAD_X), '"' + ((ok ? colors.green : colors.red)(arg.username)) + '"', "on", this.which() + ":", arg.human_url];
  118. };
  119. SocialNetwork.prototype.to_list_display = function(arg) {
  120. return arg.username;
  121. };
  122. SocialNetwork.prototype.check_proof = function(check_data_json) {
  123. return (check_data_json != null ? check_data_json.name : void 0) === this.which();
  124. };
  125. SocialNetwork.prototype.to_proof = function(arg) {
  126. return new Proof({
  127. key: this.which(),
  128. value: arg.username
  129. });
  130. };
  131. return SocialNetwork;
  132. })(Base);
  133. exports.Twitter = Twitter = (function(_super) {
  134. __extends(Twitter, _super);
  135. function Twitter() {}
  136. Twitter.prototype.get_scraper_klass = function() {
  137. return proofs.TwitterScraper;
  138. };
  139. Twitter.prototype.which = function() {
  140. return "twitter";
  141. };
  142. return Twitter;
  143. })(SocialNetwork);
  144. exports.Github = Github = (function(_super) {
  145. __extends(Github, _super);
  146. function Github() {}
  147. Github.prototype.get_scraper_klass = function() {
  148. return proofs.GithubScraper;
  149. };
  150. Github.prototype.which = function() {
  151. return "github";
  152. };
  153. return Github;
  154. })(SocialNetwork);
  155. exports.Reddit = Reddit = (function(_super) {
  156. __extends(Reddit, _super);
  157. function Reddit() {}
  158. Reddit.prototype.get_scraper_klass = function() {
  159. return proofs.RedditScraper;
  160. };
  161. Reddit.prototype.which = function() {
  162. return "reddit";
  163. };
  164. return Reddit;
  165. })(SocialNetwork);
  166. exports.HackerNews = HackerNews = (function(_super) {
  167. __extends(HackerNews, _super);
  168. function HackerNews() {}
  169. HackerNews.prototype.get_scraper_klass = function() {
  170. return proofs.HackerNewsScraper;
  171. };
  172. HackerNews.prototype.which = function() {
  173. return "hackernews";
  174. };
  175. return HackerNews;
  176. })(SocialNetwork);
  177. exports.Coinbase = Coinbase = (function(_super) {
  178. __extends(Coinbase, _super);
  179. function Coinbase() {}
  180. Coinbase.prototype.get_scraper_klass = function() {
  181. return proofs.CoinbaseScraper;
  182. };
  183. Coinbase.prototype.which = function() {
  184. return "coinbase";
  185. };
  186. return Coinbase;
  187. })(SocialNetwork);
  188. exports.Dns = Dns = (function(_super) {
  189. __extends(Dns, _super);
  190. function Dns() {}
  191. Dns.prototype.get_scraper_klass = function() {
  192. return proofs.DnsScraper;
  193. };
  194. Dns.prototype.get_sub_id = function(o) {
  195. return o.domain.toLowerCase();
  196. };
  197. Dns.prototype.to_list_display = function(o) {
  198. return this.get_sub_id(o);
  199. };
  200. Dns.prototype.format_msg = function(_arg) {
  201. var arg, color, display, ok;
  202. arg = _arg.arg, display = _arg.display, ok = _arg.ok;
  203. color = ok ? 'green' : 'red';
  204. return [(ok ? CHECK : BAD_X), "admin of the DNS zone for", colors[color](arg.domain)];
  205. };
  206. Dns.prototype.check_proof = function(check_data_json) {
  207. return check_data_json.domain != null;
  208. };
  209. Dns.prototype.to_proof = function(arg) {
  210. return new Proof({
  211. key: 'dns',
  212. value: arg.domain
  213. });
  214. };
  215. return Dns;
  216. })(Base);
  217. exports.GenericWebSite = GenericWebSite = (function(_super) {
  218. __extends(GenericWebSite, _super);
  219. function GenericWebSite() {}
  220. GenericWebSite.prototype.get_scraper_klass = function() {
  221. return proofs.GenericWebSiteScraper;
  222. };
  223. GenericWebSite.prototype.get_sub_id = function(o) {
  224. var x;
  225. return ((function() {
  226. var _i, _len, _ref1, _results;
  227. _ref1 = [o.protocol, o.hostname];
  228. _results = [];
  229. for (_i = 0, _len = _ref1.length; _i < _len; _i++) {
  230. x = _ref1[_i];
  231. _results.push(x.toLowerCase());
  232. }
  233. return _results;
  234. })()).join("//");
  235. };
  236. GenericWebSite.prototype.to_list_display = function(o) {
  237. return this.get_sub_id(o);
  238. };
  239. GenericWebSite.prototype.format_msg = function(_arg) {
  240. var arg, color, display, ok;
  241. arg = _arg.arg, display = _arg.display, ok = _arg.ok;
  242. color = !ok ? 'red' : arg.protocol === 'http:' ? 'yellow' : 'green';
  243. return [(ok ? CHECK : BAD_X), "admin of", colors[color](arg.hostname), "via", colors[color](arg.protocol.toUpperCase()), arg.human_url];
  244. };
  245. GenericWebSite.prototype.check_proof = function(check_data_json) {
  246. return (check_data_json.protocol != null) && (check_data_json.hostname != null);
  247. };
  248. GenericWebSite.prototype.to_proof = function(arg) {
  249. return new Proof({
  250. key: arg.protocol.slice(0, -1),
  251. value: arg.hostname
  252. });
  253. };
  254. return GenericWebSite;
  255. })(Base);
  256. exports.alloc = function(type, cb) {
  257. var err, o;
  258. o = alloc_stub(type);
  259. if (o != null) {
  260. o.make_scraper();
  261. } else {
  262. err = new E.ScrapeError("cannot allocate scraper of type " + type);
  263. }
  264. return cb(err, o);
  265. };
  266. exports.alloc_stub = alloc_stub = function(type) {
  267. var PT, err, klass, scraper;
  268. PT = proofs.constants.proof_types;
  269. err = scraper = null;
  270. klass = (function() {
  271. switch (type) {
  272. case PT.twitter:
  273. return Twitter;
  274. case PT.github:
  275. return Github;
  276. case PT.coinbase:
  277. return Coinbase;
  278. case PT.generic_web_site:
  279. return GenericWebSite;
  280. case PT.dns:
  281. return Dns;
  282. case PT.reddit:
  283. return Reddit;
  284. case PT.hackernews:
  285. return HackerNews;
  286. default:
  287. return null;
  288. }
  289. })();
  290. if (klass) {
  291. return new klass({});
  292. } else {
  293. return null;
  294. }
  295. };
  296. }).call(this);