import datetime import json import os.path import random import time import traceback from collections import defaultdict, deque from math import ceil import pandas as pd import telepot from celery.schedules import crontab from celery.task import periodic_task from django.core.cache import caches from django.db.models import Q from django.utils import timezone from django_redis import get_redis_connection from accounts.models import FifaAccount, FifaAccountSearch, FifaAccountLog, CloseWebAppSnipes, MuleAccounts, \ CloseWebAppTransfers, FifaAccountRequest, ConsoleBotSetting, TelegramMessage, ItemPack from futplus.celery_conf import app from sbc import FIFA_REPORT_TOKEN from sbc.bots import run_discharge_snipe, run_worker_inject, run_discharge_pod_snipe from sbc.cach_data import get_console_worker_logs from sbc.models import SBCWorker, SBCOrder, AccountCheckHealthyWorker, AccountCheckHealthyOrder, \ SBCSolverServerSideHandlerWorker, SBCType, SBCProcess, SBCSolvationModel from sbc.public_methods import new_print, sell_items from accounts.web_login_utils import logout_login from sbc.sbc_solver import SBCSolver from sbc.solvation_maker.optimize import SolvationCore from trade.models import ConsoleTradeOneCard, ConsoleTradeOneLog from utils.models import SystemLog @app.on_after_finalize.connect def sbc_periodic_tasks(sender, **kwargs): print('sbc periodic_tasks started') sender.add_periodic_task(10.0, sbc_manager.s(), name='active sbc manager') sender.add_periodic_task(30.0, console_web_sbc_manager.s(), name='active console web sbc manager') sender.add_periodic_task(10.0, mule_sell_items.s(), name='active mule sell items manager', queue='can_open_chrome') sender.add_periodic_task(20.0, account_check_healthy_worker.s(), name='start account healthy', queue='can_open_chrome') sender.add_periodic_task(10.0, account_check_healthy_worker_2.s(), name='account healthy', queue='can_open_chrome') # sender.add_periodic_task(5.0, discharge_sniper.s(), name='active discharge sniper', queue='sniper_queue') sender.add_periodic_task(10.0, discharge_pod_sniper.s(), name='active discharge pod sniper', queue='sniper_queue') # sender.add_periodic_task(60.0, transfer_injection.s(), name='transfer injection') sender.add_periodic_task(100.0, delete_old_transfers.s(), name='delete old transfers') # sender.add_periodic_task(5.0, sbc_server_side_manager.s(), name='sbc server side manager') sender.add_periodic_task(60 * 60, remove_old_logs.s(), name='remove old logs') sender.add_periodic_task(20, get_workers_data.s(), name='get workers data') sender.add_periodic_task(60 * 5, deactive_expire_sbc_types.s(), name='check expire sbc') # sender.add_periodic_task(60 * 3, update_workers_log_cache.s(), name='update log cache') # sender.add_periodic_task(5, save_prints.s(), name='save prints') sender.add_periodic_task(5, save_prints2.s(), name='save prints2') sender.add_periodic_task(60, remove_cant_buy_players.s(), name='remove cant buy players') sender.add_periodic_task(60, start_server_get_backup_code.s(), name='server get backup codes', queue='can_open_chrome') @app.task def get_workers_data(): max_concurrency1, created = ConsoleBotSetting.objects.get_or_create( name='celery_max_concurrency', defaults={'int_value': 0}) active_concurrency1, created = ConsoleBotSetting.objects.get_or_create( name='celery_active_concurrency', defaults={'int_value': 0}) max_concurrency = 0 active_concurrency = 0 workers_data = app.control.inspect().stats() if workers_data: workers_data = workers_data.values() else: return 'can not read celery data' for worker in workers_data: max_concurrency += int(worker['pool']['max-concurrency']) active_tasks = app.control.inspect().active() for item in active_tasks: active_concurrency += len(active_tasks.get(item)) max_concurrency1.int_value = max_concurrency active_concurrency1.int_value = active_concurrency max_concurrency1.save() active_concurrency1.save() active_workers = app.control.inspect().ping() active_workers_name = [] for active_worker in active_workers: active_workers_name.append(str(active_worker)) worker_object, created = ConsoleBotSetting.objects.get_or_create( name=str(active_worker), defaults={'int_value': 1}) worker_object.update_time = timezone.localtime() worker_object.save(update_fields=['update_time']) return f'max : {max_concurrency} , active : {active_concurrency} , active workers : {active_workers_name}' @app.task(bind=True, queue='can_open_chrome') def active_bot(self, uncompleted_worker_id): n = random.uniform(0.1, 2) time.sleep(n) uncompleted_worker = SBCWorker.objects.filter(id=uncompleted_worker_id).first() uncompleted_worker.task_id = self.request.id uncompleted_worker.last_run_time = timezone.localtime() uncompleted_worker.must_done = False uncompleted_worker.save() fifa_account = uncompleted_worker.fifa_account if not fifa_account.active: fifa_account.need_captcha = 0 fifa_account.active = True fifa_account.last_run_time = timezone.localtime() fifa_account.save() sbc_solver_item = SBCSolver(uncompleted_worker.id, fifa_account.id, fifa_account.user_name, fifa_account.password, fifa_account.platform, uncompleted_worker.manual_loyal) try: login_result = logout_login(sbc_solver_item, uncompleted_worker, fifa_account, ) if login_result.get('status_bool') is False and sbc_solver_item.running_platform in ['console_web', 'web']: fifa_account.refresh_from_db() fifa_account.active = False fifa_account.save() uncompleted_worker.refresh_from_db() uncompleted_worker.error = True uncompleted_worker.error_description = str(login_result.get('reason')) uncompleted_worker.save() return 'can`t login , leave it for now' except Exception as e: new_print(fifa_account, 'error on sbc active bot : ', traceback.format_exc()) # handle_end_bot() new_print(fifa_account, '##############################################') try: new_print(fifa_account, 'main_dic["persona"] = ', sbc_solver_item.persona_id) except: new_print(fifa_account, 'exception error 106 : ', traceback.format_exc()) raise Exception('error') try: sbc_solver_item.sbc_solver_core() except: new_print(fifa_account, 'exception error 119 : ', traceback.format_exc()) # sbc_solver_item.sbc_solver_core2() @app.task def remove_old_logs(): FifaAccountSearch.objects.filter(search_time__lt=timezone.localtime() - timezone.timedelta(days=10)).delete() FifaAccountLog.objects.using('logs').filter(log_time__lt=timezone.localtime() - timezone.timedelta(days=10)).delete() SystemLog.objects.filter(log_time__lt=timezone.localtime() - timezone.timedelta(days=10)).delete() FifaAccountRequest.objects.using('logs').filter(create_time__lt=timezone.localtime() - timezone.timedelta(days=10)).delete() ConsoleTradeOneCard.objects.filter(create_time__lt=timezone.localtime() - timezone.timedelta(days=30)).delete() ConsoleTradeOneLog.objects.filter(create_time__lt=timezone.localtime() - timezone.timedelta(days=30)).delete() SBCWorker.objects.filter(create_time__lt=timezone.localtime() - timezone.timedelta(days=30 * 3)).delete() SBCProcess.objects.filter(create_time__lt=timezone.localtime() - timezone.timedelta(days=30)).delete() TelegramMessage.objects.filter(create_time__lt=timezone.localtime() - timezone.timedelta(days=30 * 6)).delete() ItemPack.objects.filter(create_time__lt=timezone.localtime() - timezone.timedelta(days=30 * 12)).delete() @app.task def sbc_manager(): # print('inside sbc manager') max_concurrency = ConsoleBotSetting.objects.get(name='celery_max_concurrency').int_value active_concurrency = ConsoleBotSetting.objects.get(name='celery_active_concurrency').int_value active_workers_name = ConsoleBotSetting.objects.filter( name__startswith='celery', update_time__gte=timezone.localtime() - timezone.timedelta(minutes=1) ).values_list('name', flat=True) sbc_worker = SBCWorker.objects.filter( fifa_account__active=False, last_run_time__lt=timezone.localtime() - datetime.timedelta(hours=1, minutes=2), must_done=False, is_done=False, has_error=False, running_platform='web', fifa_account__console=None, ).first() if sbc_worker: # if active_concurrency + 12 < max_concurrency: if active_concurrency < 10 and 'celery@can_open_chrome' in active_workers_name: active_bot.delay(sbc_worker.id) # print('account %s worker started ...' % sbc_worker.fifa_account.user_name) else: sbc_worker = SBCWorker.objects.filter( fifa_account__active=True, last_run_time__lt=timezone.localtime() - datetime.timedelta(hours=5, minutes=2), must_done=False, is_done=False, has_error=False, running_platform='web', fifa_account__console=None, ).first() if sbc_worker: new_print(sbc_worker.fifa_account, 'account was active , but last run time was more than 5 hours ago') sbc_worker.fifa_account.active = False sbc_worker.fifa_account.save(update_fields=['active']) # remove_old_logs() driver_check = FifaAccount.objects.filter(driver=1).first() # print('driver check = ', driver_check) if not driver_check: # print('will kill chromes') # os.system('pkill -f chrome') # os.system("sudo pkill -f chrome") os.system( 'pgrep -f chrome | grep -v -E "can_open_chrome|sniper kill chromes|kill_chromes" | xargs sudo kill -9') # FifaAccount.objects.filter(last_search_update__lt=timezone.localtime() - datetime.timedelta(hours=24)).update( # search_number=0, last_search_update=timezone.localtime()) sbc_orders = SBCOrder.objects.filter(status=1) for sbc_order in sbc_orders: error = 0 done = 0 sbc_workers = SBCWorker.objects.filter(sbc_order=sbc_order) for sbc_worker in sbc_workers: if sbc_worker.has_error: error += 1 if sbc_worker.is_done: done += 1 if len(sbc_workers) <= error + done and len(sbc_workers) > 0: if not error: sbc_order.status = 2 elif not done: sbc_order.status = 3 else: sbc_order.status = 4 sbc_order.save(update_fields=['status']) return f'active : {active_concurrency} , max : {max_concurrency}' @app.task def console_web_sbc_manager(): sbc_worker = SBCWorker.objects.filter( is_done=False, has_error=False, must_done=False, running_platform='console_web', last_run_time__lt=timezone.localtime() - datetime.timedelta(hours=1, minutes=2), fifa_account__active=False, fifa_account__console=None, ).order_by('last_run_time').first() max_concurrency = ConsoleBotSetting.objects.get(name='celery_max_concurrency').int_value active_concurrency = ConsoleBotSetting.objects.get(name='celery_active_concurrency').int_value active_workers_name = ConsoleBotSetting.objects.filter( name__startswith='celery', update_time__gte=timezone.localtime() - timezone.timedelta(minutes=1) ).values_list('name', flat=True) if sbc_worker: # if active_concurrency + 12 < (max_concurrency): if active_concurrency < 10 and 'celery@can_open_chrome' in active_workers_name: active_bot.delay(sbc_worker.id) # remove_old_logs() # sbc_orders = SBCOrder.objects.filter(status=1) # for sbc_order in sbc_orders: # error = 0 # done = 0 # sbc_workers = SBCWorker.objects.filter(sbc_order=sbc_order) # for sbc_worker in sbc_workers: # if sbc_worker.has_error: # error += 1 # if sbc_worker.is_done: # done += 1 # if len(sbc_workers) == error + done and len(sbc_workers) > 0: # if not error: # sbc_order.status = 2 # elif not done: # sbc_order.status = 3 # else: # sbc_order.status = 4 # sbc_order.save(update_fields=['status']) return f'active : {active_concurrency} , max : {max_concurrency}' @app.task def discharge_sniper(): transfer = CloseWebAppSnipes.objects.filter( error=False, first_side_ready=True, second_side_done=False, web_app_is_on_it=False ).first() if transfer: new_print(transfer.first_account, '*' * 30, ' start discharge sniper in web ', '*' * 30) transfer.web_app_is_on_it = True transfer.save(update_fields=['web_app_is_on_it']) run_discharge_snipe(transfer) @app.task def discharge_pod_sniper(): transfer = CloseWebAppTransfers.objects.filter( error=False, for_discharge=True, first_side_done=False, web_app_is_on_it=False).first() if transfer: new_print(transfer.first_account, '*' * 30, ' start discharge pod sniper in web ', '*' * 30) transfer.web_app_is_on_it = True transfer.web_app_start_time = timezone.localtime() transfer.save(update_fields=['web_app_is_on_it', 'web_app_start_time']) run_discharge_pod_snipe(transfer) @app.task def delete_old_transfers(): transfer = CloseWebAppSnipes.objects.filter( error=False, first_side_ready=True, second_side_done=False, web_app_is_on_it=False ).first() if transfer: old_transfer = CloseWebAppSnipes.objects.filter( error=False, first_side_ready=True, second_side_done=False, web_app_is_on_it=True, id__lt=transfer.id - 100 ).first() if old_transfer: old_transfer.error = True old_transfer.first_side_done = True old_transfer.save(update_fields=['error', 'first_side_done']) time.sleep(2) MuleAccounts.objects.filter( in_use=True, last_used__lt=timezone.localtime() - datetime.timedelta(minutes=60) ).update( in_use=False ) must_close_transfer = CloseWebAppSnipes.objects.filter( error=False, first_side_done=False, second_side_done=False, insert_time__lt=timezone.localtime() - timezone.timedelta(hours=2) ).first() if must_close_transfer: must_close_transfer.error = True must_close_transfer.error_desc = 'more than 15 min' must_close_transfer.save(update_fields=['error', 'error_desc']) @app.task def transfer_injection(): transfer = CloseWebAppTransfers.objects.filter( error=False, for_inject=True, first_side_done=True, second_side_done=False, web_app_is_on_it=False ).first() if transfer: accounts = MuleAccounts.objects.filter( Q(fifa_account__delete_console_reason=None) | Q(fifa_account__delete_console_reason=''), fifa_account__credit__gt=transfer.buy_now_price, in_use=False, error=False ).order_by('last_used').all() for account in accounts: discharge_exist = CloseWebAppTransfers.objects.filter( error=False, for_discharge=True, first_side_done=False, first_account=account.fifa_account ).first() if discharge_exist: continue sbc_worker, created = SBCWorker.objects.get_or_create( fifa_account=account.fifa_account, running_platform='inject') sbc_worker.first_xi = True sbc_worker.puzzle_master = True sbc_worker.manual_loyal = True sbc_worker.save() if sbc_worker.has_error: account.error = True account.error_description = sbc_worker.error_description account.save() else: transfer.web_app_is_on_it = True transfer.web_app_start_time = timezone.localtime() transfer.save() new_print(account.fifa_account, 'mule account = ', account.fifa_account.user_name, ' will buy ', transfer.buy_now_price) account.last_used = timezone.localtime() account.in_use = True account.save() run_worker_inject(sbc_worker.id, transfer) # threading.Thread(target=run_worker_inject, # kwargs={'worker_id': sbc_worker.id, 'transfer': transfer}).start() break @app.task(bind=True) def account_check_healthy_worker(self, ): max_concurrency = ConsoleBotSetting.objects.get(name='celery_max_concurrency').int_value active_concurrency = ConsoleBotSetting.objects.get(name='celery_active_concurrency').int_value # if (active_concurrency + 12) < max_concurrency: if active_concurrency < 15: account_healthy = AccountCheckHealthyWorker.objects.filter( Q(status='') | Q(status=None), is_done=False, has_error=False, fifa_account__console=None, ).first() # if account_healthy and account_healthy.fifa_account.active is False: if account_healthy: account_healthy.status = 'doing' account_healthy.task_id = self.request.id account_healthy.save() account_healthy.fifa_account.active = True account_healthy.fifa_account.save() # running_platform = account_healthy.running_platform running_platform = 'web_check_healthy' exists_worker = SBCWorker.objects.filter( is_done=False, has_error=False, running_platform=running_platform, fifa_account=account_healthy.fifa_account, ).last() if not exists_worker: exists_worker, created = SBCWorker.objects.get_or_create( is_done=False, has_error=False, running_platform=running_platform, fifa_account=account_healthy.fifa_account, ) account_healthy.sbc_worker = exists_worker account_healthy.save() try: exists_worker.task_id = self.request.id exists_worker.save() sbc_solver = SBCSolver( worker_id=exists_worker.id, fifa_account_id=account_healthy.fifa_account.id, username=account_healthy.fifa_account.user_name, password=account_healthy.fifa_account.password, platform=account_healthy.fifa_account.platform, manual_loyal=False, running_platform=running_platform, use_public_moves=False, account_healthy_worker_id=account_healthy.id ) sbc_solver.check_healthy_utils.core( open_preview_gold_pack=account_healthy.open_preview_gold_pack, sell_all_items=account_healthy.sell_items, sell_club_items=account_healthy.sell_club_items, open_packs=account_healthy.open_packs, create_club=account_healthy.create_club, order_active_squad=account_healthy.order_active_squad, ) # account_healthy.refresh_from_db() account_healthy.is_done = True account_healthy.status = 'done' account_healthy.save(update_fields=['is_done', 'status']) except Exception as error: new_print(account_healthy.fifa_account, 'error 78 : ', str(traceback.format_exc())) # account_healthy.refresh_from_db() account_healthy.error_description = error account_healthy.has_error = True account_healthy.save(update_fields=['error_description', 'has_error']) # exists_worker.refresh_from_db() exists_worker.is_done = True exists_worker.save(update_fields=['is_done']) # account_healthy.fifa_account.refresh_from_db() account_healthy.fifa_account.active = False account_healthy.fifa_account.save(update_fields=['active']) account_healthy.refresh_from_db() if not account_healthy.is_done and not account_healthy.has_error: account_healthy.is_done = True account_healthy.save() # elif account_healthy: # account_healthy.has_error = True # account_healthy.error_description = 'Fifa Account Is Active. Deactivate it' # account_healthy.save() else: return f'max concurrency active : {active_concurrency} , max : {max_concurrency}' request_orders = AccountCheckHealthyOrder.objects.filter(status=1) for request_order in request_orders: error = 0 done = 0 request_workers = AccountCheckHealthyWorker.objects.filter(request_order=request_order) for account_healthy_worker in request_workers: if account_healthy_worker.has_error: error += 1 if account_healthy_worker.is_done: done += 1 if len(request_workers) == error + done and len(request_workers) > 0: if not error: request_order.status = 2 elif not done: request_order.status = 3 else: request_order.status = 4 request_order.save(update_fields=['status']) @app.task(bind=True) def account_check_healthy_worker_2(self, ): max_concurrency = ConsoleBotSetting.objects.get(name='celery_max_concurrency').int_value active_concurrency = ConsoleBotSetting.objects.get(name='celery_active_concurrency').int_value active_workers_name = ConsoleBotSetting.objects.filter( name__startswith='celery', update_time__gte=timezone.localtime() - timezone.timedelta(minutes=1) ).values_list('name', flat=True) # if (active_concurrency + 12) < max_concurrency: if active_concurrency < 10 and 'celery@can_open_chrome' in active_workers_name: pass else: return f'max concurrency active : {active_concurrency} , max : {max_concurrency}' sbc_worker = SBCWorker.objects.filter( is_done=False, has_error=False, must_done=False, running_platform='web_healthy', last_run_time__lt=timezone.localtime() - datetime.timedelta(hours=1, minutes=2), fifa_account__active=False, ).order_by('last_run_time').first() # if sbc_worker and sbc_worker.fifa_account.active is False: if sbc_worker: sbc_worker.status = 'doing' sbc_worker.task_id = self.request.id sbc_worker.last_run_time = timezone.localtime() sbc_worker.save(update_fields=['status', 'task_id', 'last_run_time']) sbc_worker.fifa_account.active = True sbc_worker.fifa_account.last_run_time = timezone.localtime() sbc_worker.fifa_account.save(update_fields=['active', 'last_run_time']) try: sbc_worker.task_id = self.request.id sbc_worker.save(update_fields=['task_id']) check_healthy_runner = SBCSolver( worker_id=sbc_worker.id, fifa_account_id=sbc_worker.fifa_account.id, username=sbc_worker.fifa_account.user_name, password=sbc_worker.fifa_account.password, platform=sbc_worker.fifa_account.platform, manual_loyal=False, running_platform=sbc_worker.running_platform, use_public_moves=False, ) # todo : fix bellow and uncomment # check_healthy_runner.core( # open_preview_gold_pack=account_healthy.open_preview_gold_pack, # sell_all_items=account_healthy.sell_items, # sell_club_items=account_healthy.sell_club_items, # open_packs=account_healthy.open_packs, # create_club=account_healthy.create_club, # order_active_squad=account_healthy.order_active_squad, # ) run_result = check_healthy_runner.check_healthy_utils.core( open_preview_gold_pack=True, sell_all_items=False, sell_club_items=False, open_packs=False, create_club=True, order_active_squad=False, ) sbc_worker.is_done = True sbc_worker.status = 'done' sbc_worker.description = str(run_result) sbc_worker.save(update_fields=['is_done', 'status', 'description']) except Exception as error: new_print(sbc_worker.fifa_account, 'error 78 : ', str(traceback.format_exc())) sbc_worker.error_description = error sbc_worker.has_error = True sbc_worker.save(update_fields=['error_description', 'has_error']) sbc_worker.fifa_account.active = False sbc_worker.fifa_account.save(update_fields=['active']) request_orders = SBCOrder.objects.filter(status=1, create_time__gt=timezone.localtime() - timezone.timedelta(days=3)) # for request_order in request_orders: # error = 0 # done = 0 # request_workers = SBCWorker.objects.filter(request_order=request_order) # for account_healthy_worker in request_workers: # if account_healthy_worker.has_error: # error += 1 # if account_healthy_worker.is_done: # done += 1 # if len(request_workers) == error + done and len(request_workers) > 0: # if not error: # request_order.status = 2 # elif not done: # request_order.status = 3 # else: # request_order.status = 4 # request_order.save(update_fields=['status']) if request_orders: bugged_workers = SBCWorker.objects.filter( is_done=False, has_error=False, must_done=False, running_platform='web_healthy', last_run_time__lt=timezone.localtime() - datetime.timedelta(hours=1, minutes=2), fifa_account__active=True) bugged_active_accounts = FifaAccount.objects.filter(id__in=bugged_workers.values_list('fifa_account_id', flat=True)).update(active=False) return {'bugged_active_accounts': bugged_active_accounts} @app.task def sbc_server_side_manager(): max_concurrency = ConsoleBotSetting.objects.get(name='celery_max_concurrency').int_value active_concurrency = ConsoleBotSetting.objects.get(name='celery_active_concurrency').int_value if (active_concurrency + 12) > max_concurrency: return 'max concurrency' handler_worker = SBCSolverServerSideHandlerWorker.objects.filter(is_done=False, has_error=False, status='0').first() if handler_worker: try: handler_worker.status = 1 handler_worker.save() new_print(handler_worker.fifa_account, '#' * 30, 'start sbc server side handler ') sbc_solver = SBCSolver( handler_worker.sbc_worker.id, handler_worker.fifa_account.id, handler_worker.fifa_account.user_name, handler_worker.fifa_account.password, handler_worker.fifa_account.platform, running_platform='console', manual_loyal=False, use_public_moves=False) login_result = logout_login(sbc_solver, sbc_worker=handler_worker.sbc_worker, fifa_account=handler_worker.fifa_account) if login_result.get('status_bool') is False: new_print(handler_worker.fifa_account, 'login fail , handle it , ', login_result) raise Exception(login_result.get('reason')) result = sbc_solver.sbc_solver_core() handler_worker.sbc_complete_status = str(result)[:400] new_print(sbc_solver.fifa_account, '****** end of sbc server side handler ******') handler_worker.status = 2 handler_worker.is_done = True handler_worker.save() except: handler_worker.status = 3 handler_worker.is_done = True handler_worker.has_error = True handler_worker.error_description = traceback.format_exc() handler_worker.end_time = timezone.localtime() handler_worker.save() return 'work done' @app.task def mule_sell_items(): bot = telepot.Bot(FIFA_REPORT_TOKEN) max_concurrency = ConsoleBotSetting.objects.get(name='celery_max_concurrency').int_value active_concurrency = ConsoleBotSetting.objects.get(name='celery_active_concurrency').int_value if active_concurrency > 15 or active_concurrency + 12 > max_concurrency: return 'max concurrency' discharge_with_snipe_lowest_buy_now_price = ConsoleBotSetting.objects.get( name='discharge_with_snipe_lowest_buy_now_price').int_value uncompleted_worker = SBCWorker.objects.filter( is_done=False, has_error=False, running_platform='mule_web_sell_items', last_run_time__lt=timezone.localtime() - datetime.timedelta(hours=1, minutes=2), fifa_account__active=False).first() uncompleted_workers = SBCWorker.objects.filter( is_done=False, has_error=False, running_platform='mule_web_sell_items', last_run_time__lt=timezone.localtime() - datetime.timedelta(hours=1, minutes=2), fifa_account__active=False) if uncompleted_worker: uncompleted_worker.task_id = 1 uncompleted_worker.last_run_time = timezone.localtime() uncompleted_worker.must_done = False uncompleted_worker.save() fifa_account = uncompleted_worker.fifa_account if not fifa_account.active: fifa_account.need_captcha = 0 fifa_account.active = True fifa_account.last_run_time = timezone.localtime() fifa_account.save() sbc_solver_item = SBCSolver(uncompleted_worker.id, fifa_account.id, fifa_account.user_name, fifa_account.password, fifa_account.platform, uncompleted_worker.manual_loyal) for nnnt in range(5): try: login_result = logout_login(sbc_solver_item, uncompleted_worker, fifa_account) new_print(fifa_account, 'listing mule items - login result : ', login_result) # if login_result == 'login failed': if login_result.get('status_bool') is False: uncompleted_worker.has_error = True uncompleted_worker.error_description = str(login_result.get('reason')) uncompleted_worker.save(update_fields=['has_error', 'error_description']) try: mule_acc = MuleAccounts.objects.get(fifa_account=fifa_account) mule_acc.error = True mule_acc.error_description = str(login_result.get('reason')) mule_acc.save(update_fields=['error', 'error_description']) except: new_print(fifa_account, 'error on save mule : ', traceback.format_exc()) break sell_items(parent_instance=sbc_solver_item, sbc_worker=uncompleted_worker, fifa_account=fifa_account, sell_club_players=True) check_result = sbc_solver_item.check_active_unlisted_items() active_trades = check_result.get('active_trades') sum_active_trades = check_result.get('sum_active_trades') sum_starting_bids = check_result.get('sum_starting_bids') sum_unlisted_items = check_result.get('sum_unlisted_items') # active_trades, sum_active_trades, sum_starting_bids, sum_unlisted_items = sbc_solver_item.check_active_unlisted_items() new_print(fifa_account, 'sum_starting_bids : ', sum_starting_bids, ' sum_active_trades : ', sum_active_trades) uncompleted_worker.refresh_from_db() if (sum_starting_bids < discharge_with_snipe_lowest_buy_now_price ) or (sum_active_trades < 3 and sum_starting_bids < 10000): uncompleted_worker.is_done = True uncompleted_worker.save(update_fields=['is_done']) try: from sbc.login_to_ea_with_selenium import login_to_account backup_code_obj = fifa_account.account_backup_code.last() driver, cooki, cc, bb = login_to_account( sbc_solver_item.username, sbc_solver_item.password, None, proxy_ip=None, proxy_port=None, proxy_user=None, proxy_pass=None, app_auth=backup_code_obj.app_code, cookies=fifa_account.selenium_cookies) if driver and isinstance(driver, str): raise Exception(driver) backup_codes = sbc_solver_item.get_new_backup_codes(driver, None, backup_code_obj.app_code) backup_code_obj.backup_code = backup_codes[0] backup_code_obj.backup_codes_str = ','.join(backup_codes) backup_code_obj.save(update_fields=['backup_codes_str', 'backup_code']) driver.quit() except Exception as error: new_print(fifa_account, 'cant get backup codes 2 , error : ', traceback.format_exc()) sbc_solver_item.handle_end_bot() uncompleted_worker.complete_number += 1 uncompleted_worker.save(update_fields=['complete_number']) if len(list(uncompleted_workers)) <= 6: search_24_hour = fifa_account.account_search.filter( search_time__gt=timezone.localtime() - timezone.timedelta(hours=24)).count() text = f'mule sbc done\n' \ f'{fifa_account}\n' \ f'search : {search_24_hour}\n' \ f'trade access : {fifa_account.trade_access}' sbc_solver_item.send_message(bot, 123, text) break except Exception as e: new_print(fifa_account, 'error on sbc mull sell items bot : ', traceback.format_exc()) if str(e) == 'After Handle End Bot': break fifa_account.refresh_from_db() fifa_account.active = False fifa_account.save(update_fields=['active']) return f'active : {active_concurrency} , max : {max_concurrency}' @app.task def deactive_expire_sbc_types(): expired_sbc_types = SBCType.objects.exclude( expire_time=None, ).filter( expire_time__lt=timezone.localtime() - timezone.timedelta(minutes=10), must_done__gt=0) expired_count = expired_sbc_types.count() expired_sbc_types.update(must_done=0) return {'expired sbc count': expired_count} @periodic_task(run_every=crontab(hour=20, minute=30)) def restart_last_daily_sbc(): bot = telepot.Bot(FIFA_REPORT_TOKEN) repeatable_sbcs = SBCType.objects.exclude( expire_time=None, ).filter( repeatable__in=['immediately', 'daily'] ).filter( expire_time__gt=timezone.localtime() + timezone.timedelta(hours=12), must_done=1, ) first_repeatable = repeatable_sbcs.first() if first_repeatable: # remove daily SBCs to restart sbc processes again complete_account = SBCProcess.objects.filter( sbc_type=first_repeatable, is_done=True, create_time__lt=timezone.localtime() - timezone.timedelta(minutes=30) ).distinct('worker__fifa_account').values_list('worker__fifa_account__id', flat=True).count() complete_number = SBCProcess.objects.filter( sbc_type=first_repeatable, is_done=True, create_time__lt=timezone.localtime() - timezone.timedelta(minutes=30) ).count() SBCProcess.objects.filter( Q(is_done=True) | Q(has_error=True), sbc_type__in=repeatable_sbcs, create_time__lt=timezone.localtime() - timezone.timedelta(minutes=30) ).delete() try: text = f"sbc number {first_repeatable.sbc_number} - {first_repeatable.name} \n will restart \n" \ f"completed account : {complete_account} \n complete number : {complete_number}" chat_id = '-885694083' bot.sendMessage(chat_id, text) except: pass # stop last automated workers last_orders = SBCOrder.objects.filter( file_name__in=['automated_xbox360_sbc.xlsx', 'automated_xboxs_sbc.xlsx', 'automated_failed_sbc.xlsx'], create_time__gt=timezone.localtime() - timezone.timedelta(days=2)) if last_orders: stopped_workers = SBCWorker.objects.filter( sbc_order__in=last_orders, is_done=False, has_error=False, ).update( has_error=True, error_description='automate deactivated' ) return {'result': f'{complete_account} completed this challenge', 'stopped_workers': str(stopped_workers)} else: return {'result': 'no repeatable sbc found'} @app.task def update_workers_log_cache(): get_console_worker_logs(force_update=True) @app.task(queue='sbc_solvation_queue') def get_solvation(solvation_model_dict, json_players_data, fifa_account_id, max_resolve_time=600, minimize_objective='rating'): try: # solvation_model = SBCSolvationModel.objects.get(id=solvation_model_id) solvation_model_json = json.loads(solvation_model_dict) solvation_model_json.pop('excluded_rarity_ids', []) solvation_model = SBCSolvationModel(**solvation_model_json) if fifa_account_id: fifa_account = FifaAccount.objects.get(id=fifa_account_id) new_print(fifa_account, 'solvation on server started , model : ', solvation_model) solvation = SolvationCore() solvation.set_conditions(solvation_model) if minimize_objective == 'cost': solvation.MINIMIZE_OBJECTIVE = 'cost' solvation.max_time_in_seconds = max_resolve_time if fifa_account_id: solvation.fifa_account = fifa_account data_frame = pd.read_json(json_players_data, orient='records') data_frame = data_frame.reset_index(drop=True).astype( {'Rating': 'int32', 'Cost': 'int32', 'Rarity': 'str', 'Country': 'str', 'League': 'str', 'Club': 'str'}) formation = solvation.formation_dict_2.get(solvation_model.formation).copy() final_players = solvation.solver(data_frame) if final_players: df_out = data_frame.iloc[final_players].copy() df_out.insert(5, 'Is_Pos', df_out.pop('Is_Pos')) players_targets = df_out.to_dict('records') players_targets = sorted(players_targets, key=lambda d: d['Is_Pos'], reverse=True) need_buy_player = 0 in [ie['IsExists'] for ie in players_targets] if solvation_model.required_positions: correct_positions = solvation_model.required_positions.split(',') else: correct_positions = list(solvation_model.sbc_type.sbctarget_set.values_list('position', flat=True)) return { 'status_bool': True, 'status': 'success', 'need_buy_player': need_buy_player, 'formation': formation, 'correct_positions': correct_positions, 'chemistry': solvation_model.chemistry, 'chem_per_player': solvation_model.chem_per_player, 'players_targets': players_targets, 'solve_status_text': solvation.solve_status_text, 'final_players': final_players } return {'status_bool': False, 'status': 'failed', 'solve_status_text': solvation.solve_status_text, 'formation': formation} except: return {'status_bool': False, 'status': 'failed', 'exception': traceback.format_exc()} # print_cache = caches['print_log'] # # # @app.task # def save_prints(): # # todo : if this function run every 5 seconds , must sort data by time before save # bulk_create = [] # keys = print_cache.keys('print_log_*')[:50000] # for value in dict(print_cache.get_many(keys)).values(): # bulk_create.append(FifaAccountLog( # fifa_account_id=value.get('fifa_account_id'), description=value.get('description'), # log_time=value.get('log_time'))) # print_cache.delete_many(keys) # FifaAccountLog.objects.using('logs').bulk_create(objs=bulk_create) # # bulk_create2 = [] # keys2 = print_cache.keys('request_log_*')[:50000] # for value2 in dict(print_cache.get_many(keys2)).values(): # bulk_create2.append(FifaAccountRequest( # fifa_account_id=value2.get('fifa_account_id'), link=value2.get('link'), # create_time=value2.get('create_time'))) # print_cache.delete_many(keys2) # FifaAccountRequest.objects.using('logs').bulk_create(objs=bulk_create2) print_cache2 = get_redis_connection('print_log') @app.task def save_prints2(): bulk_create = [] logs = [json.loads(log.decode('utf-8')) for log in print_cache2.lrange('print_log2', 0, 50000)] print_cache2.ltrim('print_log2', len(logs), -1) for value in logs: desc = value.get('description') if isinstance(desc, str): desc = desc.replace('\x00', '') bulk_create.append(FifaAccountLog( fifa_account_id=value.get('fifa_account_id'), description=desc, log_time=value.get('log_time'))) FifaAccountLog.objects.using('logs').bulk_create(objs=bulk_create) bulk_create2 = [] logs2 = [json.loads(log.decode('utf-8')) for log in print_cache2.lrange('request_log2', 0, 50000)] for value2 in logs2: bulk_create2.append(FifaAccountRequest( fifa_account_id=value2.get('fifa_account_id'), link=value2.get('link'), create_time=value2.get('create_time'))) print_cache2.ltrim('request_log2', len(logs2), -1) FifaAccountRequest.objects.using('logs').bulk_create(objs=bulk_create2) return {'prints': len(logs), 'requests': len(logs2)} @app.task def remove_cant_buy_players(): items = print_cache2.lrange('cant_buy_players', 0, -1) for item in items: item_data = json.loads(item) create_time = timezone.datetime.fromisoformat(item_data['create_time']) elapsed_time = timezone.localtime() - create_time if elapsed_time.total_seconds() > 3600: print_cache2.lrem('cant_buy_players', 0, item) return {'items': len(items)} @app.task def start_server_get_backup_code(): print('working on get_backup_code on pc') need_run_server_web_backup = SBCWorker.objects.filter( is_done=False, has_error=False, must_done=False, running_platform='server_web_backup_code', fifa_account__active=False, ) need_backup_count = need_run_server_web_backup.count() print('need backup code for ', need_backup_count) if need_backup_count: item = need_run_server_web_backup.first() uncompleted_worker = SBCWorker.objects.filter(id=item.id).first() uncompleted_worker.task_id = '1' uncompleted_worker.last_run_time = timezone.localtime() uncompleted_worker.must_done = False uncompleted_worker.save(update_fields=['task_id', 'last_run_time', 'must_done']) fifa_account = uncompleted_worker.fifa_account new_print(fifa_account, 'start console back up code ', timezone.localtime()) # if not fifa_account.active: fifa_account.need_captcha = 0 fifa_account.active = True fifa_account.last_run_time = timezone.localtime() fifa_account.save(update_fields=['need_captcha', 'active', 'last_run_time']) sbc_solver_item = SBCSolver(uncompleted_worker.id, fifa_account.id, fifa_account.user_name, fifa_account.password, fifa_account.platform, uncompleted_worker.manual_loyal) backup_code_obj = fifa_account.account_backup_code.last() if not backup_code_obj: new_print(fifa_account, 'bot stop , no app code found') uncompleted_worker.has_error = True uncompleted_worker.error_description = 'no app code found' uncompleted_worker.save(update_fields=['has_error', 'error_description']) return from sbc.login_to_ea_with_selenium import login_to_account proxy_obj = fifa_account.proxy login_selenium_result = login_to_account( sbc_solver_item.username, sbc_solver_item.password, None, proxy_ip=proxy_obj.ip_address, proxy_port=proxy_obj.port, proxy_user=proxy_obj.user_name, proxy_pass=proxy_obj.password, app_auth=backup_code_obj.app_code, cookies=fifa_account.selenium_cookies) if login_selenium_result.get('status_bool') is True: driver = login_selenium_result.get('driver') backup_codes = sbc_solver_item.get_new_backup_codes(driver, None, backup_code_obj.app_code) backup_code_obj.backup_code = backup_codes[0] backup_code_obj.backup_codes_str = ','.join(backup_codes) backup_code_obj.save(update_fields=['backup_codes_str']) uncompleted_worker.is_done = True uncompleted_worker.save(update_fields=['is_done']) driver.quit() else: new_print(fifa_account, 'cant login 2, error : ', traceback.format_exc()) uncompleted_worker.has_error = True uncompleted_worker.error_description = login_selenium_result.get('error') uncompleted_worker.save(update_fields=['has_error', 'error_description']) @periodic_task(run_every=crontab(hour=21, minute=50)) def create_automated_series_sbc(): return sbc_workers = [] sbc_order = SBCOrder.objects.create(file_name='automated_xboxs_sbc.xlsx', status=1) raw_accounts = FifaAccount.objects.filter( Q(delete_console_reason=None) | Q(delete_console_reason='') | Q(delete_console_reason='no_console'), console__is_active=True, platform__in=['xboxs', 'xbox360'] ).values_list('id', 'console_id', 'platform').order_by('console_id', 'platform', 'id') grouped = defaultdict(list) for acc_id, console_id, platform in raw_accounts: grouped[(console_id, platform)].append(acc_id) first_half_groups = [] second_half = [] for (console_id, platform), acc_ids in grouped.items(): if platform == 'xbox360': mid = ceil(len(acc_ids) / 2) else: mid = len(acc_ids) // 2 first_half_groups.append((console_id, platform, acc_ids[:mid])) second_half.extend(acc_ids[mid:]) queues = { (console_id, platform): deque(acc_ids) for console_id, platform, acc_ids in first_half_groups if acc_ids } ordered_ids = [] keys = list(queues.keys()) iio = 0 while queues: key = keys[iio % len(keys)] if key in queues: queue = queues[key] if queue: ordered_ids.append(queue.popleft()) if not queue: del queues[key] keys.remove(key) iio -= 1 iio += 1 for fifa_account_id in ordered_ids: sbc_workers.append(SBCWorker( fifa_account_id=fifa_account_id, sbc_order=sbc_order, last_run_time=timezone.localtime() - datetime.timedelta(hours=4), status='waiting for sbc', status_change_time=timezone.localtime() - datetime.timedelta(hours=1), manual_loyal=False, running_platform='console_web_pc', order_active_squad=False, order_active_squad_formation=False, )) SBCWorker.objects.bulk_create(sbc_workers) sbc_order.workers_count = len(sbc_workers) sbc_order.save(update_fields=['workers_count']) return {'status': f'Xbox Series SBC File Added {len(sbc_workers)}'} @periodic_task(run_every=crontab(hour=21, minute=55)) def create_automated_360_sbc(): return sbc_workers = [] sbc_order = SBCOrder.objects.create(file_name='automated_xbox360_sbc.xlsx', status=1) raw_accounts = FifaAccount.objects.filter( Q(delete_console_reason=None) | Q(delete_console_reason=''), console__is_active=True, platform__in=['xboxs', 'xbox360'] ).values_list('id', 'console_id', 'platform').order_by('console_id', 'platform', 'id') grouped = defaultdict(list) for acc_id, console_id, platform in raw_accounts: grouped[(console_id, platform)].append(acc_id) first_half = [] second_half_groups = [] for (console_id, platform), acc_ids in grouped.items(): if platform == 'xbox360': mid = ceil(len(acc_ids) / 2) else: mid = len(acc_ids) // 2 first_half.extend(acc_ids[:mid]) second_half_groups.append((console_id, platform, acc_ids[mid:])) queues = { (console_id, platform): deque(acc_ids) for console_id, platform, acc_ids in second_half_groups if acc_ids } ordered_ids = [] keys = list(queues.keys()) iio = 0 while queues: key = keys[iio % len(keys)] if key in queues: queue = queues[key] if queue: ordered_ids.append(queue.popleft()) if not queue: del queues[key] keys.remove(key) iio -= 1 iio += 1 for fifa_account_id in ordered_ids: sbc_workers.append(SBCWorker( fifa_account_id=fifa_account_id, sbc_order=sbc_order, last_run_time=timezone.localtime() - datetime.timedelta(hours=4), status='waiting for sbc', status_change_time=timezone.localtime() - datetime.timedelta(hours=1), manual_loyal=False, running_platform='console_web_pc', order_active_squad=False, order_active_squad_formation=False, )) SBCWorker.objects.bulk_create(sbc_workers) sbc_order.workers_count = len(sbc_workers) sbc_order.save(update_fields=['workers_count']) return {'status': f'Xbox 360 SBC File Added {len(sbc_workers)}'} @periodic_task(run_every=crontab(hour=14, minute=30)) def failed_daily_sbc_order(): last_orders = SBCOrder.objects.filter( file_name__in=['automated_xbox360_sbc.xlsx', 'automated_xboxs_sbc.xlsx'], create_time__gte=timezone.localtime() - timezone.timedelta(hours=24) ) failed_accounts = SBCWorker.objects.filter( sbc_order__in=last_orders, error_description__in=[ 'ea servers unavailable', 'worker stopped middle of work', 'login failed', 'problem in login', 'bad information' ], ).distinct('fifa_account__id').values_list('fifa_account__id', flat=True) sbc_workers = [] sbc_order = SBCOrder.objects.create(file_name='automated_failed_sbc.xlsx', status=1) for fifa_account in FifaAccount.objects.filter( Q(delete_console_reason=None) | Q(delete_console_reason='') | Q(delete_console_reason='no_console'), console__is_active=True, id__in=failed_accounts ): sbc_workers.append(SBCWorker( fifa_account=fifa_account, sbc_order=sbc_order, last_run_time=timezone.localtime() - datetime.timedelta(hours=4), status='waiting for sbc', status_change_time=timezone.localtime() - datetime.timedelta(hours=1), manual_loyal=False, running_platform='console_web_pc', order_active_squad=False, order_active_squad_formation=False, )) SBCWorker.objects.bulk_create(sbc_workers) sbc_order.workers_count = len(sbc_workers) sbc_order.save(update_fields=['workers_count']) return {'success': True, 'created_workers': len(sbc_workers)}