diff --git a/.dockerignore b/.dockerignore index b85fe55..f55198e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,10 @@ ./data/logs/*.logs ./data/data.db +./data/profiles/prof_* .vscode .gitignore .idea/ ./__pycache__/ -./GlobalExambBot/__pycache__/ \ No newline at end of file +./GlobalExambBot/__pycache__/ +.git +*.md \ No newline at end of file diff --git a/GlobalExamBot/Sheets.py b/GlobalExamBot/Sheets.py index 507ada3..ba4566f 100644 --- a/GlobalExamBot/Sheets.py +++ b/GlobalExamBot/Sheets.py @@ -21,11 +21,11 @@ class Sheets: WebDriverWait(self.driver, 15).until(ec.visibility_of_element_located((By.XPATH, self.pagecard_xpath))) page_cards = self.driver.find_elements(by=By.XPATH, value=self.pagecard_xpath) card_list = [] - for card in page_cards : + for card in page_cards : if not self.manageSheets.link_exist(card.get_attribute('href')): card_list.append(card) return card_list - + def watch(self, Sheets_el): self.actions.move_to_element(Sheets_el).click(Sheets_el).perform() WebDriverWait(self.driver, 15).until(ec.visibility_of_element_located((By.XPATH, self.Sheetscard_xpath))) diff --git a/GlobalExamBot/bot.py b/GlobalExamBot/bot.py index 77cd3d1..9e72f33 100644 --- a/GlobalExamBot/bot.py +++ b/GlobalExamBot/bot.py @@ -1,5 +1,6 @@ import logging import os +import sys from GlobalExamBot.helpers import TypeInField, element_exists, wait_between from GlobalExamBot.Sheets import Sheets @@ -11,6 +12,7 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.common.keys import Keys class Bot: + def __init__(self, driver, action, configuration): self.driver = driver self.actions = action @@ -18,7 +20,7 @@ class Bot: self.email_xpath = '//input[@name="email"]' self.password_xpath = '//input[@name="password"]' self.index = 0 - self.scrollcount = 0 + self.catindex = 0 self.categories = ['https://exam.global-exam.com/library/study-sheets/categories/grammar', 'https://exam.global-exam.com/library/study-sheets/categories/language-functions', 'https://exam.global-exam.com/library/study-sheets/categories/vocabulary'] @@ -33,7 +35,7 @@ class Bot: password_el.send_keys(Keys.RETURN) def run(self): - + profile = f'prof_{self.configuration.username}' if not os.path.exists(f'./Profiles/{profile}'): @@ -49,20 +51,17 @@ class Bot: Sheets_action = Sheets(self.driver, self.actions, self.configuration) while True: - self.driver.get('https://exam.global-exam.com/library/study-sheets/categories/grammar') + self.driver.get(self.categories[self.catindex]) Sheets_list = Sheets_action.search() if Sheets_list : logging.info(f'Sheets n°{ self.index }') Sheets_action.watch(Sheets_list[0]) self.index +=1 - self.scrollcount = 0 wait_between(3,10) else: - logging.info('All visible Sheets have already been read. Need to scroll down ...') - self.driver.execute_script(f"window.scrollTo(0, document.body.scrollHeight)") - self.scrollcount += 1 - wait_between(5,15) - if self.scrollcount > 10: - logging.info('End of page or network error.') - self.scrollcount = 0 - logging.info(self.driver.get_log('browser')) \ No newline at end of file + if self.catindex != len(self.categories) - 1 : + logging.info('All visible Sheets have already been read. Use of the next category.') + self.catindex += 1 + else: + logging.info('No category available.') + sys.exit(1) diff --git a/GlobalExamBot/constants.py b/GlobalExamBot/constants.py index 2b263db..7bfc66e 100644 --- a/GlobalExamBot/constants.py +++ b/GlobalExamBot/constants.py @@ -1,4 +1,4 @@ def init(): global VERSION, APP_NAME APP_NAME = 'GlobalExamBot' - VERSION = '1.0.0' \ No newline at end of file + VERSION = '1.0.0' diff --git a/GlobalExamBot/database.py b/GlobalExamBot/database.py index a3be9c0..9add543 100644 --- a/GlobalExamBot/database.py +++ b/GlobalExamBot/database.py @@ -1,6 +1,9 @@ import sqlite3 class Database: + """ + Database management + """ def __init__(self, database_link='./data/data.db'): """ Database constructor @@ -18,7 +21,7 @@ class Database: c.execute('''INSERT INTO sheet_links (link) VALUES (:link);''', (link,)) c.close() connection.commit() - + def link_exist(self, link): """ Returns true if the link exists in the database. @@ -45,4 +48,4 @@ def create_table_sheets(): c.execute('''CREATE TABLE IF NOT EXISTS sheet_links (id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE, link text);''') c.close() - connection.commit() \ No newline at end of file + connection.commit() diff --git a/GlobalExamBot/driver.py b/GlobalExamBot/driver.py index 9296519..f6b3de0 100644 --- a/GlobalExamBot/driver.py +++ b/GlobalExamBot/driver.py @@ -3,6 +3,9 @@ from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.chrome.options import Options class Driver: + """ + Browser management + """ def __init__(self, profile): self.profile = profile self.chrome_options = None @@ -10,6 +13,9 @@ class Driver: self.action = None def setup(self, log_path='./data/logs/', headless=True): + """ + Browser configuration + """ self.chrome_options = Options() # Anti bot detection @@ -22,7 +28,7 @@ class Driver: # Maximize Browser self.chrome_options.add_argument('--start-maximized') - + # Headless Mode if headless: self.chrome_options.add_argument('--headless') @@ -44,7 +50,6 @@ class Driver: self.chrome_options.add_argument("--disable-low-res-tiling") self.chrome_options.add_argument("--log-level=3") self.chrome_options.add_argument("--silent") - # Disable save password prefs = {'credentials_enable_service': False, @@ -54,12 +59,12 @@ class Driver: # Set profile self.chrome_options.add_argument(f'user-data-dir=./data/profiles/{self.profile}') - self.driver = webdriver.Chrome(f'./ChromeDriver/chromedriver',options=self.chrome_options, service_args=[f'--log-path={log_path}ChromeDriver.log']) + self.driver = webdriver.Chrome('./ChromeDriver/chromedriver',options=self.chrome_options, service_args=[f'--log-path={log_path}ChromeDriver.log']) self.action = ActionChains(self.driver) return self.driver, self.action - + def get_driver(self): return self.driver - + def get_action(self): - return self.action \ No newline at end of file + return self.action diff --git a/GlobalExamBot/helpers.py b/GlobalExamBot/helpers.py index 6552000..8879ca9 100644 --- a/GlobalExamBot/helpers.py +++ b/GlobalExamBot/helpers.py @@ -38,7 +38,7 @@ class Helpers: def load_configuration(self): """ this method allows you to load arguments. - :return: args + :return: args """ header() # Load all configuration variables @@ -50,10 +50,13 @@ class Helpers: return args def logging_configuration(self, logging_level=logging.INFO, filename='data/logs/bot_globalexam.log'): + """ + Log configuration + """ logging.basicConfig(filename=filename, level=logging_level, format='%(asctime)s - %(levelname)s - %(message)s') - + root_logger = logging.getLogger() root_logger.setLevel(logging_level) @@ -74,26 +77,26 @@ def header(): logging.info('==\t version : ' + const.VERSION + ' \t==') logging.info('==\t=============================================================\t==') -def wait_between( min, max): +def wait_between( minimum, maximum): """ - Wait random time in second beetween min and max seconds, to have an not linear behavior and be more human. + Wait random time in second beetween min and max seconds, + to have an not linear behavior and be more human. """ - rand=uniform(min, max) + rand=uniform(minimum, maximum) sleep(rand) -def TypeInField(element, xpath, myValue): +def TypeInField(element, xpath, value): """Type in a field""" - val = myValue elem = element.find_element(by=By.XPATH, value=xpath) - for i in range(len(val)): - elem.send_keys(val[i]) + for ele in enumerate(value): + elem.send_keys(ele[1]) wait_between(0.2, 0.4) wait_between(0.4, 0.7) def element_exists(xpath, element, by=By.XPATH): """ Check if an element exist - + :return: Boolean """ try: @@ -101,4 +104,4 @@ def element_exists(xpath, element, by=By.XPATH): element.find_element(by=by, value=xpath) except: return False - return True \ No newline at end of file + return True diff --git a/main.py b/main.py index bc3d071..6b9bba3 100644 --- a/main.py +++ b/main.py @@ -8,14 +8,14 @@ from GlobalExamBot.database import create_table_sheets from GlobalExamBot.driver import Driver from GlobalExamBot.bot import Bot - + def main(): try: helpers = Helpers() # Configuration of the logging library helpers.logging_configuration() - + # Load all configuration variables config = helpers.load_configuration() @@ -24,7 +24,7 @@ def main(): create_table_sheets() profile = f'prof_{config.username}' - + logging.info(f'Username : {config.username}') # Initialize driver and actions @@ -46,4 +46,4 @@ def main(): sys.exit(1) if __name__ == "__main__": - main() \ No newline at end of file + main()