461 lines
17 KiB
Python
461 lines
17 KiB
Python
#!/usr/bin/env python
|
|
#-*- coding: UTF-8 -*-
|
|
|
|
#Searching and Downloading Google Images/Image Links
|
|
|
|
#Import Libraries
|
|
|
|
import time #Importing the time library to check the time of code execution
|
|
import sys #Importing the System Library
|
|
|
|
import os
|
|
import os.path as osp
|
|
|
|
import fnmatch
|
|
|
|
import json
|
|
|
|
#import urllib2
|
|
from urllib import urlencode
|
|
|
|
from urllib2 import Request,urlopen
|
|
from urllib2 import URLError, HTTPError
|
|
|
|
import datetime
|
|
|
|
import hashlib
|
|
|
|
from collections import OrderedDict
|
|
|
|
########### CONFIGS ###########
|
|
# Path to config_file
|
|
config_file = './config.json'
|
|
|
|
########### Default CONFIGS ###########
|
|
CONFIGS = {}
|
|
|
|
# How many images you want to download for each class. Google will only return 100 images at most for one search
|
|
CONFIGS[u'num_downloads_for_each_class'] = 200
|
|
|
|
# image type to search
|
|
CONFIGS[u'search_file_type'] = 'jpg'
|
|
#CONFIGS[u'search_file_type'] = 'bmp'
|
|
#CONFIGS[u'search_file_type'] = 'png'
|
|
|
|
# Because google only returns at most 100 results for each search query,
|
|
# we must send many search queries to get more than 100 results.
|
|
# We need to set cdr (date range, in the form of "tbs=cdr:1,cd_min:{start_date},cd_max:{end_date}") to tell google
|
|
# we want to search images in some date range (start_date, end_date),
|
|
# so as to get different results for each search query.
|
|
# CONFIGS[u'search_cdr_days'] is the days between cd_min and cd_max.
|
|
CONFIGS[u'search_cdr_days'] = 60
|
|
|
|
#This dict is used to search keywords. You can edit this dict to search for google images of your choice. You can simply add and remove elements of the list.
|
|
#{class1:[list of related keywords], class2:[list of related keywords]...}
|
|
CONFIGS[u'search_keywords_dict'] = {'animal':[u'猫', 'cat', 'dog'],
|
|
'fruit':[u'apple', u'banaba']}
|
|
|
|
#This list is used to further add suffix to your search term. Each element of the list will help you download 100 images. First element is blank which denotes that no suffix is added to the search keyword of the above list. You can edit the list by adding/deleting elements from it.So if the first element of the search_keyword is 'Australia' and the second element of keywords is 'high resolution', then it will search for 'Australia High Resolution'
|
|
#aux_keywords = [' high resolution']
|
|
|
|
CONFIGS[u'save_dir'] = './downloads'
|
|
|
|
CONFIGS[u'output_prefix'] = 'download_urls'
|
|
CONFIGS[u'output_suffix'] = 'google'
|
|
|
|
print '==>Default CONFIGS:'
|
|
print CONFIGS
|
|
########### End of Default CONFIGS ###########
|
|
|
|
########### Load config.json if there is one ###########
|
|
if osp.exists(config_file):
|
|
print "Load CONFIGS from " + config_file
|
|
fp = open(config_file, 'a+')
|
|
CONFIGS_loaded = json.load(fp, object_pairs_hook=OrderedDict)
|
|
|
|
print '==>Loaded CONFIGS:'
|
|
print CONFIGS_loaded
|
|
|
|
for k,v in CONFIGS_loaded.iteritems():
|
|
if k in CONFIGS:
|
|
CONFIGS[k] = v
|
|
|
|
fp.close()
|
|
|
|
print '==>CONFIGS after loading:'
|
|
print CONFIGS
|
|
########### End of Load config.json ###########
|
|
|
|
|
|
#CONFIGS[u'output_prefix'] = CONFIGS[u'output_prefix'] + '_'
|
|
#CONFIGS[u'output_suffix'] = '_' + CONFIGS[u'output_suffix']
|
|
|
|
CONFIGS[u'save_dir'] = CONFIGS[u'save_dir']+'/'
|
|
if not osp.exists(CONFIGS[u'save_dir']):
|
|
os.mkdir(CONFIGS[u'save_dir'])
|
|
|
|
########### End of CONFIGS ###########
|
|
|
|
########### Functions to Load downloaded urls ###########
|
|
def load_url_files(_dir, file_name_prefix):
|
|
url_list = []
|
|
|
|
ttl_url_list_file_name = osp.join(_dir, file_name_prefix +'_all.txt')
|
|
if osp.exists(ttl_url_list_file_name):
|
|
fp_urls = open(ttl_url_list_file_name, 'r') #Open the text file called database.txt
|
|
print 'load URLs from file: ' + ttl_url_list_file_name
|
|
|
|
i = 0
|
|
for line in fp_urls:
|
|
line = line.strip()
|
|
if len(line)>0:
|
|
splits = line.split('\t')
|
|
url_list.append(splits[0].strip())
|
|
i=i+1
|
|
|
|
print str(i) + ' URLs loaded'
|
|
fp_urls.close()
|
|
else:
|
|
url_list = load_all_url_files(_dir, file_name_prefix)
|
|
|
|
return url_list
|
|
|
|
def load_all_url_files(_dir, file_name_prefix):
|
|
url_list = []
|
|
|
|
for file_name in os.listdir(_dir):
|
|
if fnmatch.fnmatch(file_name, file_name_prefix +'*.txt'):
|
|
file_name = osp.join(_dir, file_name)
|
|
fp_urls = open(file_name, 'r') #Open the text file called database.txt
|
|
print 'load URLs from file: ' + file_name
|
|
|
|
i = 0
|
|
for line in fp_urls:
|
|
line = line.strip()
|
|
if len(line)>0:
|
|
splits = line.split('\t')
|
|
url_list.append(splits[0].strip())
|
|
i=i+1
|
|
print str(i) + ' URLs loaded'
|
|
fp_urls.close()
|
|
|
|
return url_list
|
|
########### End of Functions to Load downloaded urls ###########
|
|
|
|
############## Functions to get date/time strings ############
|
|
def get_current_date():
|
|
tm = time.gmtime()
|
|
date = datetime.date(tm.tm_year, tm.tm_mon, tm.tm_mday)
|
|
return date
|
|
|
|
def get_new_date_by_delta_days(date, delta_days):
|
|
delta = datetime.timedelta(delta_days)
|
|
new_date = date+delta
|
|
return new_date
|
|
|
|
#Make a string from current GMT time
|
|
def get_gmttime_string():
|
|
_str = time.strftime("GMT%Y%m%d_%H%M%S", time.gmtime())
|
|
return _str
|
|
|
|
#Make a string from current local time
|
|
def get_localtime_string():
|
|
_str = time.strftime("%Y%m%d_%H%M%S", time.localtime())
|
|
return _str
|
|
############## End of Functions to get date/time strings ############
|
|
|
|
############## Google Image Search functions ############
|
|
# Get Image URL list form Google image search by keyword
|
|
def google_get_query_url(keyword, file_type, cdr):
|
|
url = None
|
|
|
|
# if keyword is unicode, we need to encode it into utf-8
|
|
if isinstance(keyword, unicode):
|
|
keyword = keyword.encode('utf-8')
|
|
|
|
query = dict(q = keyword,
|
|
tbm = 'isch',
|
|
tbs=cdr+',ift:'+file_type)
|
|
|
|
#url = 'https://www.google.com/search?q=' + keyword + '&espv=2&biw=1366&bih=667&site=webhp&source=lnms&tbm=isch&sa=X&ei=XosDVaCXD8TasATItgE&ved=0CAcQ_AUoAg'
|
|
#url = 'https://www.google.com/search?as_oq=' + keyword + '&as_st=y&tbm=isch&safe=images&tbs=ift:jpg'
|
|
url = 'https://www.google.com/search?'+urlencode(query)
|
|
|
|
print "\t==>Google Query URL is: " + url
|
|
return url
|
|
|
|
#Downloading entire Web Document (Raw Page Content)
|
|
def google_download_page(url):
|
|
version = (3,0)
|
|
cur_version = sys.version_info
|
|
if cur_version >= version: #If the Current Version of Python is 3.0 or above
|
|
import urllib.request #urllib library for Extracting web pages
|
|
try:
|
|
headers = {}
|
|
headers['User-Agent'] = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
|
|
req = urllib.request.Request(url, headers = headers)
|
|
resp = urllib.request.urlopen(req)
|
|
respData = str(resp.read())
|
|
return respData
|
|
except Exception as e:
|
|
print(str(e))
|
|
else: #If the Current Version of Python is 2.x
|
|
import urllib2
|
|
try:
|
|
headers = {}
|
|
headers['User-Agent'] = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
|
|
req = urllib2.Request(url, headers = headers)
|
|
response = urllib2.urlopen(req)
|
|
page = response.read()
|
|
return page
|
|
except:
|
|
return"Page Not found"
|
|
|
|
#Finding 'Next Image' from the given raw page
|
|
def google_images_get_next_item(s):
|
|
start_line = s.find('rg_di')
|
|
if start_line == -1: #If no links are found then give an error!
|
|
end_quote = 0
|
|
link = "no_links"
|
|
return link, end_quote
|
|
else:
|
|
start_line = s.find('"class="rg_meta"')
|
|
start_content = s.find('"ou"',start_line+1)
|
|
end_content = s.find(',"ow"',start_content+1)
|
|
content_raw = str(s[start_content+6:end_content-1])
|
|
return content_raw, end_content
|
|
|
|
#Getting all links with the help of '_images_get_next_image'
|
|
def google_images_get_all_items(page):
|
|
items = []
|
|
while True:
|
|
item, end_content = google_images_get_next_item(page)
|
|
if item == "no_links":
|
|
break
|
|
else:
|
|
items.append(item) #Append all the links in the list named 'Links'
|
|
time.sleep(0.1) #Timer could be used to slow down the request for image downloads
|
|
page = page[end_content:]
|
|
return items
|
|
|
|
def google_search_keyword(keyword, file_type, cdr):
|
|
query_url = google_get_query_url(keyword, file_type, cdr)
|
|
raw_html = (google_download_page(query_url))
|
|
time.sleep(0.1)
|
|
image_url_list = google_images_get_all_items(raw_html)
|
|
return image_url_list
|
|
############## End of Google Image Search functions ############
|
|
|
|
############## Functions to get real urls and download images ############
|
|
#Get real url of a input url
|
|
def get_real_url(url, loaded_urls):
|
|
real_url = None
|
|
response = None
|
|
try:
|
|
req = Request(url, headers={"User-Agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"})
|
|
response = urlopen(req)
|
|
|
|
real_url = response.geturl()
|
|
print 'Real_url is: ' + str(real_url)
|
|
|
|
if real_url in loaded_urls:
|
|
print 'URL had been downloaded in previous '
|
|
real_url = None
|
|
|
|
except IOError as e: #If there is any IOError
|
|
print("IOError on url "+str(url))
|
|
print e
|
|
except HTTPError as e: #If there is any HTTPError
|
|
print("HTTPError on url "+str(url))
|
|
print e
|
|
except URLError as e:
|
|
print("URLError on url "+str(url))
|
|
print e
|
|
|
|
if response:
|
|
response.close()
|
|
|
|
return real_url
|
|
|
|
def download_image(url, save_dir, loaded_urls=None):
|
|
real_url = None
|
|
response = None
|
|
save_image_name = None
|
|
try:
|
|
req = Request(url, headers={"User-Agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"})
|
|
response = urlopen(req)
|
|
|
|
real_url = response.geturl()
|
|
|
|
if loaded_urls and real_url in loaded_urls:
|
|
print 'URL had been downloaded in previous searching'
|
|
real_url = None
|
|
else:
|
|
img_name = hashlib.md5(real_url).hexdigest()
|
|
save_image_name = save_dir + '/' + img_name + '.' + CONFIGS[u'search_file_type']
|
|
print 'Try to save image ' + real_url + ' into file: ' + save_image_name
|
|
output_file = open(save_image_name,'wb')
|
|
data = response.read()
|
|
output_file.write(data)
|
|
|
|
#response.close()
|
|
except IOError as e: #If there is any IOError
|
|
print("IOError on url "+str(url))
|
|
print e
|
|
except HTTPError as e: #If there is any HTTPError
|
|
print("HTTPError on url "+str(url))
|
|
print e
|
|
except URLError as e:
|
|
print("URLError on url "+str(url))
|
|
print e
|
|
|
|
if response:
|
|
response.close()
|
|
|
|
return real_url, save_image_name
|
|
############## End of Functions to get real urls and download images ############
|
|
|
|
############## Main Program ############
|
|
t0 = time.time() #start the timer
|
|
|
|
#Download Image Links
|
|
i= 0
|
|
|
|
cur_date = get_current_date()
|
|
print "Today is: " + cur_date.strftime("%Y/%m/%d")
|
|
|
|
time_str = get_gmttime_string()
|
|
|
|
for class_name,search_keywords in CONFIGS[u'search_keywords_dict'].iteritems():
|
|
print "Class no.: " + str(i+1) + " -->" + " Class name = " + str(class_name)
|
|
|
|
class_urls_file_prefix = CONFIGS[u'output_prefix'] + '_' + str(class_name).strip()
|
|
|
|
items = load_url_files(CONFIGS[u'save_dir'], class_urls_file_prefix)
|
|
loaded_urls_num = len(items)
|
|
print 'Loaded URLs in total is: ', loaded_urls_num
|
|
|
|
# load pre-saved download parameters, actually cd_min for date range
|
|
cd_max = cur_date
|
|
|
|
params_file = osp.join(CONFIGS[u'save_dir'], class_urls_file_prefix + '_params_' + CONFIGS[u'output_suffix'] + '.txt')
|
|
print 'Loaded pre-saved download parameters from: ' + params_file
|
|
params_list = []
|
|
fp_params = open(params_file, 'a+')
|
|
for line in fp_params:
|
|
line = line.strip()
|
|
if line!='':
|
|
params_list.append(line)
|
|
print "\t-->loaded parameters: ", line
|
|
|
|
if len(params_list)>0:
|
|
splits = params_list[-1].split('/')
|
|
if len(splits)==3:
|
|
cd_max = datetime.date(int(splits[0]), int(splits[1]), int(splits[2]))
|
|
|
|
cd_min = get_new_date_by_delta_days(cd_max, -CONFIGS[u'search_cdr_days'])
|
|
print 'cd_max: ', cd_max
|
|
print 'cd_min: ', cd_min
|
|
|
|
print ("Crawling Images...")
|
|
|
|
class_save_dir = osp.join(CONFIGS[u'save_dir'], class_urls_file_prefix + '_' + time_str + '_' + CONFIGS[u'output_suffix'])
|
|
if not osp.exists(class_save_dir):
|
|
os.mkdir(class_save_dir)
|
|
|
|
output_all_urls_file = osp.join(CONFIGS[u'save_dir'], class_urls_file_prefix +'_all.txt')
|
|
fp_all_urls = open(output_all_urls_file, 'a+')
|
|
|
|
output_urls_file = osp.join(CONFIGS[u'save_dir'], class_urls_file_prefix + '_' + time_str + '_' + CONFIGS[u'output_suffix'] + '.txt')
|
|
fp_urls = open(output_urls_file, 'a+')
|
|
|
|
# if osp.exists(output_urls_file):
|
|
# fp_urls = open(output_urls_file, 'a+') #Open the text file called database.txt
|
|
# for line in fp_urls:
|
|
# items.append(line.strip())
|
|
# else:
|
|
# fp_urls = open(output_urls_file, 'w+') #Open the text file called database.txt
|
|
#
|
|
cdr_enabled = False
|
|
|
|
while True:
|
|
if cdr_enabled:
|
|
cdr = 'cdr:1,cd_min:{},cd_max:{}'.format(cd_min.strftime('%m/%d/%Y'), cd_max.strftime('%m/%d/%Y'))
|
|
print "==>Search for Images between " + cd_min.strftime("%Y/%m/%d") + \
|
|
' and ' + cd_max.strftime("%Y/%m/%d")
|
|
else:
|
|
cdr = ''
|
|
print "==>Search for Images in any time"
|
|
|
|
j = 0
|
|
|
|
# Google only return 100 images at most for one search. So we may need to try many times
|
|
while j<len(search_keywords):
|
|
print "\t==>Class name=" + str(class_name) + ', search keywords=' + search_keywords[j]
|
|
keyword = search_keywords[j]#.replace(' ','%20')
|
|
|
|
# # if keyword is unicode, we need to encode it into utf-8
|
|
# if isinstance(keyword, unicode):
|
|
# keyword = keyword.encode('utf-8')
|
|
|
|
# query = dict(q = keyword,
|
|
# tbm = 'isch',
|
|
# tbs=tbs+',ift:'+CONFIGS[u'search_file_type'])
|
|
#
|
|
# #url = 'https://www.google.com/search?q=' + keyword + '&espv=2&biw=1366&bih=667&site=webhp&source=lnms&tbm=isch&sa=X&ei=XosDVaCXD8TasATItgE&ved=0CAcQ_AUoAg'
|
|
# #url = 'https://www.google.com/search?as_oq=' + keyword + '&as_st=y&tbm=isch&safe=images&tbs=ift:jpg'
|
|
# url = 'https://www.google.com/search?'+urlencode(query)
|
|
#
|
|
# print "\t==>Query URL is: " + url
|
|
#
|
|
# raw_html = (download_page(url))
|
|
# time.sleep(0.1)
|
|
# new_items = _images_get_all_items(raw_html)
|
|
|
|
new_items = google_search_keyword(keyword, CONFIGS[u'search_file_type'], cdr)
|
|
|
|
for url in new_items:
|
|
#real_url = get_real_url(url)
|
|
real_url, save_name = download_image(url, class_save_dir, items)
|
|
|
|
if real_url and real_url not in items:
|
|
items.append(real_url)
|
|
fp_all_urls.write(real_url + '\t' + save_name + "\n")
|
|
fp_urls.write(real_url + '\t' + save_name + "\n")
|
|
|
|
fp_all_urls.flush()
|
|
fp_urls.flush()
|
|
|
|
print 'len(items)=', len(items)
|
|
j = j + 1
|
|
|
|
if cdr_enabled:
|
|
fp_params.write('{}/{}/{}\n'.format( cd_min.year, cd_min.month, cd_min.day))
|
|
cd_max = cd_min
|
|
cd_min = get_new_date_by_delta_days(cd_max, -CONFIGS[u'search_cdr_days'])
|
|
else:
|
|
fp_params.write('{}/{}/{}\n'.format( cd_max.year, cd_max.month, cd_max.day))
|
|
cdr_enabled = True
|
|
|
|
fp_params.flush()
|
|
|
|
print 'len(items)=', len(items)
|
|
if len(items) >= loaded_urls_num + CONFIGS[u'num_downloads_for_each_class']:
|
|
break
|
|
|
|
fp_params.close()
|
|
fp_all_urls.close()
|
|
fp_urls.close()
|
|
|
|
#print ("Image Links = "+str(items))
|
|
print ("Total New Image Links = " + str(len(items) - loaded_urls_num))
|
|
print ("\n")
|
|
i = i+1
|
|
|
|
t1 = time.time() #stop the timer
|
|
total_time = t1-t0 #Calculating the total time required to crawl, find and download all the links of 60,000 images
|
|
print("Total time taken: "+str(total_time)+" Seconds")
|
|
|
|
print("\n")
|
|
print("===All are downloaded")
|
|
#----End of the main program ----# |