diff options
25 files changed, 259 insertions, 0 deletions
diff --git a/NjuSpider/njubbs/njubbs/__init__.py b/NjuSpider/njubbs/njubbs/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/__init__.py diff --git a/NjuSpider/njubbs/njubbs/__init__.pyc b/NjuSpider/njubbs/njubbs/__init__.pyc Binary files differnew file mode 100644 index 0000000..1b1455b --- /dev/null +++ b/NjuSpider/njubbs/njubbs/__init__.pyc diff --git a/NjuSpider/njubbs/njubbs/__pycache__/__init__.cpython-34.pyc b/NjuSpider/njubbs/njubbs/__pycache__/__init__.cpython-34.pyc Binary files differnew file mode 100644 index 0000000..c403ab5 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/__pycache__/__init__.cpython-34.pyc diff --git a/NjuSpider/njubbs/njubbs/__pycache__/items.cpython-34.pyc b/NjuSpider/njubbs/njubbs/__pycache__/items.cpython-34.pyc Binary files differnew file mode 100644 index 0000000..64081e6 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/__pycache__/items.cpython-34.pyc diff --git a/NjuSpider/njubbs/njubbs/__pycache__/settings.cpython-34.pyc b/NjuSpider/njubbs/njubbs/__pycache__/settings.cpython-34.pyc Binary files differnew file mode 100644 index 0000000..ad47ee5 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/__pycache__/settings.cpython-34.pyc diff --git a/NjuSpider/njubbs/njubbs/items.py b/NjuSpider/njubbs/njubbs/items.py new file mode 100644 index 0000000..1857063 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/items.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +# Define here the models for your scraped items +# +# See documentation in: +# http://doc.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class njubbsItem(scrapy.Item): + title = scrapy.Field() + time = scrapy.Field() + author = scrapy.Field() + # define the fields for your item here like: + # name = scrapy.Field() + pass + +class QuoteItem(scrapy.Item): + text = scrapy.Field() + author = scrapy.Field() diff --git a/NjuSpider/njubbs/njubbs/items.pyc b/NjuSpider/njubbs/njubbs/items.pyc Binary files differnew file mode 100644 index 0000000..425f698 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/items.pyc diff --git a/NjuSpider/njubbs/njubbs/pipelines.py b/NjuSpider/njubbs/njubbs/pipelines.py new file mode 100644 index 0000000..69a7f7a --- /dev/null +++ b/NjuSpider/njubbs/njubbs/pipelines.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- + +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html + + +class NjubbsPipeline(object): + def process_item(self, item, spider): + return item diff --git a/NjuSpider/njubbs/njubbs/settings.py b/NjuSpider/njubbs/njubbs/settings.py new file mode 100644 index 0000000..cdec22d --- /dev/null +++ b/NjuSpider/njubbs/njubbs/settings.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +# Scrapy settings for njubbs project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# http://doc.scrapy.org/en/latest/topics/settings.html +# http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html +# http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html + +BOT_NAME = 'njubbs' + +SPIDER_MODULES = ['njubbs.spiders'] +NEWSPIDER_MODULE = 'njubbs.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = 'njubbs (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = False +# ROBOTSTXT_OBEY = True + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +#DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'njubbs.middlewares.MyCustomSpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# 'njubbs.middlewares.MyCustomDownloaderMiddleware': 543, +#} + +# Enable or disable extensions +# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html +#ITEM_PIPELINES = { +# 'njubbs.pipelines.SomePipeline': 300, +#} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See http://doc.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/NjuSpider/njubbs/njubbs/settings.pyc b/NjuSpider/njubbs/njubbs/settings.pyc Binary files differnew file mode 100644 index 0000000..d3ab4e4 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/settings.pyc diff --git a/NjuSpider/njubbs/njubbs/spiders/2 b/NjuSpider/njubbs/njubbs/spiders/2 new file mode 100644 index 0000000..51833d2 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/spiders/2 @@ -0,0 +1,21 @@ +import json +import scrapy +from njubbs.items import njubbsItem + + +class njubbsSpider(scrapy.Spider): + name = "njubbs" + start_urls = [ + 'http://bbs.nju.edu.cn/cache/t_act.js', + ] + + def parse(self, response): + strRe = "\"".join(response.text.split("\'")) + print(strRe) + strRe = strRe[5:-1] + print(strRe) + # print(response.text) + for ncd in response.xpath('//div[@id=".p.ncd__act"]'): + item = njubbsItem() + yield ncd.xpath('td') + diff --git a/NjuSpider/njubbs/njubbs/spiders/__init__.py b/NjuSpider/njubbs/njubbs/spiders/__init__.py new file mode 100644 index 0000000..ebd689a --- /dev/null +++ b/NjuSpider/njubbs/njubbs/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/NjuSpider/njubbs/njubbs/spiders/__init__.pyc b/NjuSpider/njubbs/njubbs/spiders/__init__.pyc Binary files differnew file mode 100644 index 0000000..5aa3307 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/spiders/__init__.pyc diff --git a/NjuSpider/njubbs/njubbs/spiders/__pycache__/__init__.cpython-34.pyc b/NjuSpider/njubbs/njubbs/spiders/__pycache__/__init__.cpython-34.pyc Binary files differnew file mode 100644 index 0000000..63868d8 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/spiders/__pycache__/__init__.cpython-34.pyc diff --git a/NjuSpider/njubbs/njubbs/spiders/__pycache__/jwSpider.cpython-34.pyc b/NjuSpider/njubbs/njubbs/spiders/__pycache__/jwSpider.cpython-34.pyc Binary files differnew file mode 100644 index 0000000..4aa8868 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/spiders/__pycache__/jwSpider.cpython-34.pyc diff --git a/NjuSpider/njubbs/njubbs/spiders/__pycache__/njubbsSpider.cpython-34.pyc b/NjuSpider/njubbs/njubbs/spiders/__pycache__/njubbsSpider.cpython-34.pyc Binary files differnew file mode 100644 index 0000000..70284f9 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/spiders/__pycache__/njubbsSpider.cpython-34.pyc diff --git a/NjuSpider/njubbs/njubbs/spiders/__pycache__/quoteSpider.cpython-34.pyc b/NjuSpider/njubbs/njubbs/spiders/__pycache__/quoteSpider.cpython-34.pyc Binary files differnew file mode 100644 index 0000000..0fd02c9 --- /dev/null +++ b/NjuSpider/njubbs/njubbs/spiders/__pycache__/quoteSpider.cpython-34.pyc diff --git a/NjuSpider/njubbs/njubbs/spiders/jwSpider.py b/NjuSpider/njubbs/njubbs/spiders/jwSpider.py new file mode 100644 index 0000000..6bf39ba --- /dev/null +++ b/NjuSpider/njubbs/njubbs/spiders/jwSpider.py @@ -0,0 +1,28 @@ +import scrapy +from njubbs.items import njubbsItem + +class njubbsSpider(scrapy.Spider): + name = "njujw" + start_urls = [ + 'http://jw.nju.edu.cn/', + ] + + def parse(self, response): + # print(response.text) + news = response.xpath('//div[@class="conbox1"]/div[@class="con1"]/ul/li') + print(news) + for li in news: + title = li.xpath('a/@title').extract_first() + print('\n') + url = li.xpath('a/@href').extract_first() + print(title) + print(response.urljoin(url)) + print("\n") + + + def articleParse(self, response): + title = response.xpath('//div[@class="div_detail"]/div[@class="div_title"]/center').extract_first() + + print(title) + contengt = response.xpath('//div[@class="div_detail"]/dev[@class="div_contentDetail"]') + diff --git a/NjuSpider/njubbs/njubbs/spiders/njubbsSpider.py b/NjuSpider/njubbs/njubbs/spiders/njubbsSpider.py new file mode 100644 index 0000000..de761cd --- /dev/null +++ b/NjuSpider/njubbs/njubbs/spiders/njubbsSpider.py @@ -0,0 +1,22 @@ +import json +import scrapy +from njubbs.items import njubbsItem + + +class njubbsSpider(scrapy.Spider): + name = "njubbs" + start_urls = [ + 'http://bbs.nju.edu.cn/cache/t_act.js', + ] + + def parse(self, response): + strRe = "\"".join(response.text.split("\'")) + print(strRe) + strRe = strRe[5:-24] + jsonStr = json.JSONEncoder().encode(strRe) + print(strRe) + # print(response.text) + for ncd in response.xpath('//div[@id=".p.ncd__act"]'): + item = njubbsItem() + yield ncd.xpath('td') + diff --git a/NjuSpider/njubbs/njubbs/spiders/quoteSpider.py b/NjuSpider/njubbs/njubbs/spiders/quoteSpider.py new file mode 100644 index 0000000..66f86ea --- /dev/null +++ b/NjuSpider/njubbs/njubbs/spiders/quoteSpider.py @@ -0,0 +1,14 @@ +import scrapy +from njubbs.items import QuoteItem +class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] +def parse(self, response): + for quote in response.xpath('//div[@class="quote"]'): + item = QuoteItem() + item['text'] = quote.xpath('span[@class="text"]/text()').extract_first() + item['author'] = quote.xpath('span/small/text()').extract_first() + yield item diff --git a/NjuSpider/njubbs/scrapy.cfg b/NjuSpider/njubbs/scrapy.cfg new file mode 100644 index 0000000..0aba25d --- /dev/null +++ b/NjuSpider/njubbs/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.org/en/latest/deploy.html + +[settings] +default = njubbs.settings + +[deploy] +#url = http://localhost:6800/ +project = njubbs diff --git a/NjuSpider/python b/NjuSpider/python Binary files differnew file mode 100755 index 0000000..ef0c6ab --- /dev/null +++ b/NjuSpider/python diff --git a/dotfiles/.vimperatorrc b/dotfiles/.vimperatorrc index 4f10d45..1235c65 100644 --- a/dotfiles/.vimperatorrc +++ b/dotfiles/.vimperatorrc @@ -1,5 +1,12 @@ +colorscheme simple-highlight + +set hintchars=hjklasdfgyuiopqwertnmzxcvb set gui=nonavigation let mapleader = "," +set runtimepath+=~/.vimperator/vimperator-plugins + +" Ctrl-a +inoremap <C-a> <C-Home><C-S-End> noremap d <C-d> noremap e <C-u> diff --git a/dotfiles/simple-highlight.vimp b/dotfiles/simple-highlight.vimp new file mode 100644 index 0000000..482b2c8 --- /dev/null +++ b/dotfiles/simple-highlight.vimp @@ -0,0 +1,15 @@ +" ==Vimperator_Color_Scheme== +" name: Simple Highlight +" modifications: Travis Jeffery <t.jeffery@utoronto.ca> +" modifications: Steve Lee <me@xiangyangli.com> +" - only set the hint no background; +" ==Simple_Highlight_Color_Scheme== + +highlight Bell display: none; + + +" highlight WarningMsg +highlight Hint font-family: "DejaVu Sans Mono", monospace; font-size: 12px; font-weight: normal; color: #FFO099; background-color: #3399FF; padding: 0px 2px; +highlight HintElem background-color: #00CC000; +" highlight HintActive + diff --git a/draft/socks_on_linux_app.md b/draft/socks_on_linux_app.md new file mode 100644 index 0000000..9ec8c15 --- /dev/null +++ b/draft/socks_on_linux_app.md @@ -0,0 +1,14 @@ +using tsocks + +apt-get install tsocks + +/etc/tsocks.conf +server = 127.0.0.1 +server_type = 5 +server_port = 1080 + +usage: + +tsocks wget url +tsocks pear +tsocks gems |