#!/usr/bin/env python
# -*- coding: utf-8 -*-
# ****************************************************************************
# Copyright (C) 2010 yangyingchao@gmail.com
# Author: yangyingchao <yangyingchao@gmail.com>
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2, or (at your option) any later
# version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
# You should have received a copy of the GNU General Public License along with
# GNU Emacs; see the file COPYING. If not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# ****************************************************************************
from copy import deepcopy
from sgmllib import SGMLParser
from xml.dom.minidom import *
import os
import re
import sys
import urllib2
title = "Untitled"
class MyParser(SGMLParser):
def __init__(self):
self.data = ""
self.links = []
self.TAG_BEG = False
self.TAG_END = False
SGMLParser.__init__(self, 0)
def handle_data(self, data):
if (self.TAG_BEG is True) and (self.TAG_END is False):
self.data += data
pass
def start_title(self, attrs):
self.link = ""
self.data=""
self.TAG_BEG = True
self.TAG_END = False
for (key, val) in attrs:
if key == "href":
self.link = val
def end_title(self):
self.TAG_BEG = False
self.TAG_END = True
self.title = self.data.strip()
def flush(self):
pass
def handle_comment(self, data):
pass
def start_a(self, attrs):
self.data=""
self.TAG_BEG = True
self.TAG_END = False
for (key, val) in attrs:
if key == "href":
self.link = val
def end_a(self):
self.TAG_BEG = False
self.TAG_END = True
tmp = {}
tmp["name"] = self.data
tmp["link"] = self.link
self.links.append(deepcopy(tmp))
def unknown_starttag(self, tag, attrs):
pass
def unknown_endtag(self, tag):
pass
def unknown_entityref(self, ref):
pass
def unknown_charref(self, ref):
pass
def unknown_decl(self, data):
pass
def close(self):
SGMLParser.close(self)
self.flush()
def lst2str(lst):
string = ""
for item in lst:
string += item.strip()+ "\n"
return string
def downURL(url, filename):
print "Download %s, save as %s"%(url, filename)
try:
fp = urllib2.urlopen(url)
except:
print "download exception"
print sys.exc_info()
return 0
op = open(filename, "wb")
while 1:
s = fp.read()
if not s:
break
op.write(s)
fp.close( )
op.close( )
return 1
def reptile(base_url):
"""
Download all articles from base_url.
Arguments:
- `base_url`: Url of website.
"""
page_list = []
if not len(base_url):
print "No page to reptile!"
sys.exit(1)
parser = MyParser()
if base_url.startswith("http"):
myopen = urllib2.urlopen
else:
myopen = open
try:
content = myopen(base_url).read()
except:
print "Failed to read from %s."%base_url
print sys.exc_info()
for item in content:
parser.feed(item)
for tmp in parser.links:
page_list.append(tmp.get("link"))
global title
title = parser.title
parser.close()
item_list = list(set(page_list))
for item in item_list:
# Strip '#' from url.
pos = item.find('#')
if pos != -1:
item = item[:pos]
# Added base_url to item if necessary
if not item.startswith("http"):
item = base_url.rstrip("/")+"/"+item
pass
local_file = item.split("/")[-1]
print item, local_file
if not local_file:
print "Empty local file! Continue from next one!"
continue
if os.access(local_file, os.F_OK):
print "File: %s existed, skip ..."%local_file
else:
ret = downURL(item, local_file)
# Remember to download the index file!
downURL(base_url, "index.html")
print "Total: %d articles."%(len(item_list))
pass
def walk_dir(lst, dirname, filenames):
for filename in filenames:
fn = os.path.join(dirname, filename)
if os.path.isdir(fn) or \
not filename.endswith("html"):
continue
print "Processing: %s"%fn
tmp = {}
parser = MyParser()
content = open(fn).read()
for item in content:
parser.feed(item)
tmp["file"] = filename
tmp["title"] = parser.title
parser.close()
lst.append(deepcopy(tmp))
pass
def gen_index():
"""
Generate index of all htmls in this directory.
"""
file_lists = []
os.path.walk(".", walk_dir, file_lists)
fp = open("%s.devhelp2"%os.path.basename(os.getcwd()), "w")
string = '<?xml version="1.0" encoding="utf-8"?>\n<book author=""' +\
' language="c" link="index.html" name="" title="%s"'%title+\
' version="2" xmlns="http://www.devhelp.net/book">\n <chapters>'
for item in file_lists:
link = item.get("file")
try:
name =item.get("title").decode('gbk').encode('utf-8')
except:
name = item.get("title")
finally:
string += '<sub link="%s" name="%s"/>\n'%(link, name)
string += '\n</chapters>\n </book>\n'
fp.write(string)
fp.close()
if __name__ == '__main__':
if len(sys.argv) != 2:
print "Usage: %s url of baidu space"%sys.argv[0]
print "Such as: %s http://hi.baidu.com/Username"
gen_index()
sys.exit(1)
base_url = sys.argv[1]
reptile (base_url)
gen_index()
#//python/1161