#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymongo.errors import ConfigurationError
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, WebDriverException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from pyquery import PyQuery
from urllib3.exceptions import NewConnectionError, MaxRetryError
from config_vip import *
from multiprocessing import Pool
from selenium.webdriver.chrome.options import Options
import os
import pymongo
import requests
import hashlib
import time
'''
遇到不懂的问题？Python学习交流群：821460695满足你的需求，资料都已经上传群文件，可以自行下载！
'''
if browser_method == 0:
    browser = webdriver.Chrome()
    print('你选择使用Chrome()方法...')
elif browser_method == 1:
    browser = webdriver.PhantomJS(service_args=['--load-images=false', '--disk-cache=false'])
    print('你选择使用PhantomJS()方法...')
else:
    chrome_option = Options()
    chrome_option.add_argument('--headless')
    browser = webdriver.Chrome(options=chrome_option)
    print('你选择使用Headless()方法...')

browser.set_window_size(1920, 1080)
wait = WebDriverWait(browser, 10)

try:
    client = pymongo.MongoClient(mongo_url)
    database = client[mongo_database]
except TypeError:
    print('数据库创建失败'.center(130, '*'))
except ConfigurationError:
    print('数据库创建失败'.center(130, '*'))


# 实现数据库对象

def drop_down_scrollbar():
    # 定义下拉滚动条方法
    times = 1
    while times < total_times:
        js = "var q=document.documentElement.scrollTop={}".format(times * size)
        browser.execute_script(js)
        time.sleep(1)
        times += 1


def get_search(search_word):
    # 定义get_()search方法
    url = main_url
    browser.get(url)
    # 打开url，获得内容
    time.sleep(3)
    try:
        search_bar = wait.until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '#J-search > div.c-search-form > input')))
        enter_button = wait.until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, '#J-search > div.c-search-form > a > span')))
        # 确定输入框和搜索按钮可用
        search_bar.send_keys(search_word)
        time.sleep(1)
        enter_button.click()
        # 输入关键字并点击搜索
        time.sleep(5)
        drop_down_scrollbar()
        pages = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#J_pagingCt > a:nth-child(6)')))
        # 获得总页数，main()中作为for循环参数
        print('搜索到{}共{}页的内容'.format(search_word, pages.text))
        time.sleep(3)
        print('开始获取{}第{}页的内容...'.format(search_word, str(1)))
        get_page_detail(search_word)
        print('完成获取{}第{}页的内容...'.format(search_word, str(1)))
        return pages.text
    except TimeoutException:
        print('网页未加载完成，无法搜索信息！', TimeoutException.args)
        pass
    except WebDriverException:
        print(WebDriverException.args)
        pass


def get_next_page(search_word, page):
    # 定义get_next_page()方法进行跳转
    try:
        url1 = url_search.format(search_word, str(page))
        # 找出网页规律，定个模板
        print('开始获取{}第{}页的内容...\n'.format(search_word, page))
        browser.get(url1)
        drop_down_scrollbar()
        get_page_detail(search_word)
        print('完成获取{}第{}页的内容...\n'.format(search_word, page))
    except TimeoutException:
        print('跳转网页超时！', TimeoutException.args)
        pass
    except WebDriverException:
        print(WebDriverException.args)
        pass


def get_page_detail(search_word):
    # 定义get_page_detail()方法获取网页详细信息
    try:
        source = browser.page_source
        html = PyQuery(source)
        print('解析数据成功'.center(130, '*'))
        # PyQuery解析源代码
        good_items = html('.goods-list .goods-list-item').items()
        # 调用items()方法获得数据
        for item in good_items:
            goods = {
                'good-title': item.find('.goods-title-info ').text().split('\n')[1],
                'good-sells-price': item.find('.goods-info .goods-price-wrapper .goods-sells-price .price').text(),
                'good-market-price': item.find('.goods-info  .goods-market-price').text()[2:],
                'good-discount': item.find('.goods-info .goods-discount').text(),
                'good-brand': item.find('.goods-info .goods-brand').text(),
                'image': 'http:{}'.format(item.find('.goods-slide .goods-image-link .J_img').attr('src')),
                'detail': 'http:{}'.format(item.find(' .goods-slide .goods-image-link').attr('href'))
            }
            image_url = goods['image']
            content = get_image_content(image_url)
            if content:
                # 确定图片网页是否可以打开
                download_image(content, search_word, image_url)
            save_to_mongodb(goods, search_word)
        # 调用find方法和CSS取得数据
    except TimeoutException:
        print('爬取网页超时！', TimeoutException.args)
        pass


def save_to_mongodb(goods, database_table):
    # 定义save_to_mongoDB(goods)方法将数据存储到mongoDB数据
    try:
        if database[database_table].insert(goods):
            # 插入数据成功
            print('存储数据成功'.center(130, '*'))
            print(goods, '\n')
    except Exception:
        print('写入数据出错！', Exception.args)
        pass


def get_image_content(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.content
        else:
            print('请求图片链接失败！')
    except ConnectionError:
        print(ConnectionError.args)
        return False
    except NewConnectionError:
        print(NewConnectionError.args)
        return False
    except MaxRetryError:
        print(MaxRetryError.args)
        return False


def download_image(content, folder, image_url):
    # 定义download_image(content)保存图片
    time_stamp = time.strftime("%Y%m%d", time.localtime())
    path = file_path.format(mongo_database, time_stamp, folder)
    if os.path.exists(path):
        pass
    else:
        os.makedirs(path)
    # 利用hash算法获得content MD5值以16进制显示
    filename = hashlib.md5(content).hexdigest()
    with open(file_type.format(path, filename), 'wb')as f:
        f.write(content)
        f.close()
    # 打开文件保存路径，文件名，格式，wb写入形式
    print(' {} 下载图片成功'.format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())).center(125, '*'))
    print(filename, image_url)


def main(search_word):
    pages = int(get_search(search_word))
    page = 2
    if pages >= end:
        pages = end
    try:
        while page <= pages:
            get_next_page(search_word, page)
            page += 1
    except TimeoutException:
        print(TimeoutException.args)
        pass


if __name__ == '__main__':
    pool = Pool(processes=2)
    pool.map(main, [keyword for keyword in keywords])
    pool.close()
    #锁定进程池
    pool.join()
    os.system('taskkill /im chromedriver.exe /F')
    os.system('taskkill /im chrome.exe /F')
    #杀死多余的chromedriver进程以及chrome进程

mongo_url = 'localhost'
mongo_database = 'vip'
#数据库地址以及名称
main_url = 'https://www.vip.com/'

total_times =16
size =500
#设定下来滚动条的次数和大小

browser_method = 2
#驱动浏览器的方法

start=1
end = 45
#设定结束网页，有些网页没有内容，容易引起一场
url_search = 'https://category.vip.com/suggest.php?keyword={}&page={}&count=100&suggestType=brand#catPerPos'
#定义模板
file_path = 'H:/Python_download/{}/{}/image/{}/'
file_type = '{}{}.jpg'
# 文件类型以及文件夹


keywords=['苹果','雪梨','香蕉']