From 434858b5f092534a833a915c3a2067b14969334d Mon Sep 17 00:00:00 2001 From: vg Date: Fri, 1 Jul 2022 22:55:43 +0200 Subject: git-sync on dita --- TODO.rst | 5 + gamechest.glade | 527 +++++++++++++++++++++++++++++++++++++++ gamechest.glade~ | 527 +++++++++++++++++++++++++++++++++++++++ gamechest.py | 6 + gamechest/__init__.py | 217 ++++++++++++++++ gamechest/cli.py | 66 +++++ gamechest/conf.py | 25 ++ gamechest/gamemanager.py | 183 ++++++++++++++ gamechest/gamemanager.py.backup | 431 ++++++++++++++++++++++++++++++++ gamechest/gamemanager_method1.py | 252 +++++++++++++++++++ gamechest/stepper.py | 69 +++++ gamechest/steps_install.py | 109 ++++++++ gamechest/steps_remove.py | 31 +++ gamechest/utils.py | 19 ++ 14 files changed, 2467 insertions(+) create mode 100644 TODO.rst create mode 100644 gamechest.glade create mode 100644 gamechest.glade~ create mode 100755 gamechest.py create mode 100644 gamechest/__init__.py create mode 100644 gamechest/cli.py create mode 100644 gamechest/conf.py create mode 100644 gamechest/gamemanager.py create mode 100644 gamechest/gamemanager.py.backup create mode 100644 gamechest/gamemanager_method1.py create mode 100644 gamechest/stepper.py create mode 100644 gamechest/steps_install.py create mode 100644 gamechest/steps_remove.py create mode 100644 gamechest/utils.py diff --git a/TODO.rst b/TODO.rst new file mode 100644 index 0000000..6c7ca4f --- /dev/null +++ b/TODO.rst @@ -0,0 +1,5 @@ +- implem with asyncio for steppers to simplify implementation +- with cli just run asyncio.run(main()) +- with gui, run gtk in a thread and asyncio.run in main thread (or the + reverse) and make them communicate with a queue. +- if not easily working, use https://blogs.gnome.org/jamesh/2019/08/05/glib-integration-for-python-asyncio/ diff --git a/gamechest.glade b/gamechest.glade new file mode 100644 index 0000000..603f10f --- /dev/null +++ b/gamechest.glade @@ -0,0 +1,527 @@ + + + + + + Game Characteristics + + + Game Description + + + + + + + + + + + + + False + No game in list yet + 0 + + + + + False + org.devys.apps.gamechest.svg + + + + + + + True + False + vertical + + + True + False + + + True + False + True + True + True + True + + + True + False + game_characteristics + + + 1 + 0 + + + + + True + False + False + word + fill + game_description + + + 0 + 1 + 2 + + + + + True + False + org.devys.apps.gamechest.svg + + + + 0 + 0 + + + + + 1 + 2 + + + + + True + True + 5 + 5 + gamelist_store + + + + + + + + Installed + + + + 0 + + + + + + + Title + + + + 1 + + + + + + + 0 + 2 + + + + + True + True + Search Pattern + + + 0 + 1 + + + + + True + False + Selected Game Title + + + + + + + 1 + 1 + + + + + True + True + edit-find-symbolic + False + False + Profile + + + 0 + 0 + + + + + True + False + + + + + + True + False + + + True + False + Install Progression + True + + + 1 + 0 + + + + + True + False + Global Install Progression + True + + + 1 + 1 + + + + + install + True + True + True + + + + 0 + 0 + + + + + update + True + True + True + + + 0 + 1 + + + + + remove + True + True + True + + + 0 + 2 + + + + + + + + + + + + + + + + + False + True + 1 + + + + + gtk-cancel + True + True + True + True + + + + False + True + 2 + + + + + + + + 1 + 0 + + + + + True + True + 0 + + + + + True + False + True + + + + + + run + True + True + True + + + False + True + 1 + + + + + quit + True + True + True + + + + False + True + 2 + + + + + False + True + 2 + + + + + + + False + + + + + + True + False + vertical + + + True + False + True + + + True + False + vertical + + + True + True + Search Pattern + + + False + True + 0 + + + + + True + True + + + + + + False + True + 1 + + + + + False + True + 0 + + + + + True + False + vertical + + + True + False + Selected Game Title + + + False + True + 0 + + + + + True + False + True + + + True + False + gtk-missing-image + + + False + True + 0 + + + + + True + True + + + False + True + 1 + + + + + True + True + 1 + + + + + True + True + + + True + True + 2 + + + + + False + True + 1 + + + + + True + True + 0 + + + + + True + False + True + + + install + True + True + True + + + False + True + 0 + + + + + run + True + True + True + + + False + True + 1 + + + + + quit + True + True + True + + + False + True + 2 + + + + + False + True + 2 + + + + + + diff --git a/gamechest.glade~ b/gamechest.glade~ new file mode 100644 index 0000000..0549937 --- /dev/null +++ b/gamechest.glade~ @@ -0,0 +1,527 @@ + + + + + + Game Characteristics + + + Game Description + + + + + + + + + + + + + False + No game in list yet + 0 + + + + + False + org.devys.apps.gamechest.svg + + + + + + + True + False + vertical + + + True + False + + + True + False + True + True + True + True + + + True + False + game_characteristics + + + 1 + 0 + + + + + True + False + False + word + fill + game_description + + + 0 + 1 + 2 + + + + + True + False + org.devys.apps.gamechest.svg + + + + 0 + 0 + + + + + 1 + 2 + + + + + True + True + 5 + 5 + gamelist_store + + + + + + + + Installed + + + + 0 + + + + + + + Title + + + + 1 + + + + + + + 0 + 2 + + + + + True + True + Search Pattern + + + 0 + 1 + + + + + True + False + Selected Game Title + + + + + + + 1 + 1 + + + + + True + True + edit-find-symbolic + False + False + Profile + + + 0 + 0 + + + + + True + False + + + + + + True + False + + + True + False + Install Progression + True + + + 1 + 0 + + + + + True + False + Global Install Progression + True + + + 1 + 1 + + + + + install + True + True + True + + + + 0 + 0 + + + + + update + True + True + True + + + 0 + 1 + + + + + remove + True + True + True + + + 0 + 2 + + + + + + + + + + + + + + + + + False + True + 1 + + + + + gtk-cancel + True + True + True + True + + + + False + True + 2 + + + + + + + + 1 + 0 + + + + + True + True + 0 + + + + + True + False + True + + + + + + run + True + True + True + + + False + True + 1 + + + + + quit + True + True + True + + + + False + True + 2 + + + + + False + True + 2 + + + + + + + False + + + + + + True + False + vertical + + + True + False + True + + + True + False + vertical + + + True + True + Search Pattern + + + False + True + 0 + + + + + True + True + + + + + + False + True + 1 + + + + + False + True + 0 + + + + + True + False + vertical + + + True + False + Selected Game Title + + + False + True + 0 + + + + + True + False + True + + + True + False + gtk-missing-image + + + False + True + 0 + + + + + True + True + + + False + True + 1 + + + + + True + True + 1 + + + + + True + True + + + True + True + 2 + + + + + False + True + 1 + + + + + True + True + 0 + + + + + True + False + True + + + install + True + True + True + + + False + True + 0 + + + + + run + True + True + True + + + False + True + 1 + + + + + quit + True + True + True + + + False + True + 2 + + + + + False + True + 2 + + + + + + diff --git a/gamechest.py b/gamechest.py new file mode 100755 index 0000000..8e4281d --- /dev/null +++ b/gamechest.py @@ -0,0 +1,6 @@ +#!/usr/bin/python3 + +import gamechest + +if __name__ == '__main__': + gamechest.main() diff --git a/gamechest/__init__.py b/gamechest/__init__.py new file mode 100644 index 0000000..8629624 --- /dev/null +++ b/gamechest/__init__.py @@ -0,0 +1,217 @@ +#!python3 + +''' +Usage: gamechest.py --gamelist=GAMELIST.YAML --gamedir=GAMEDIR +''' + +import logging +import os +from logging import debug, info, warning, critical + + +import docopt +import yaml +import gi +from . import gamemanager + + +gi.require_version("Gtk", "3.0") +from gi.repository import Gtk as gtk, GLib as glib, GdkPixbuf as gdkpixbuf +from gi.repository.GdkPixbuf import Pixbuf + + +class GameList: + + def __init__(self, gamelist, gamedir): + debug(f'managing gamedir {gamedir}') + debug(f'reading gamelist from {gamelist}') + with open(gamelist) as stream: + self.yaml_data = yaml.safe_load(stream) + self.gamedir = gamedir + debug('loaded data: %d bytes', len(self.yaml_data)) + + +class Handler: + + def __init__(self, builder, gamelist, gamelist_dir, gamedir): + self.builder = builder + self.gamelist = gamelist + self.gamelist_dir = gamelist_dir + self.gamedir = gamedir + repository_host = gamelist.yaml_data['repository']['host'] + repository_path = gamelist.yaml_data['repository']['path'] + self.game_installer = gamemanager.GameInstaller( + repository_host, repository_path, gamedir) + self.game_remover = gamemanager.GameRemover( + repository_host, repository_path, gamedir) + self.game_status = gamemanager.GameStatus( + repository_host, repository_path, gamedir) + + self.game_title = builder.get_object('game_title') + self.game_image = builder.get_object('game_image') + self.game_characteristics = builder.get_object('game_characteristics') + self.game_description = builder.get_object('game_description') + self.btn_install = builder.get_object('btn_install') + self.btn_update = builder.get_object('btn_update') + self.btn_remove = builder.get_object('btn_remove') + self.game_install_stop_btn = builder.get_object('game_install_stop_btn') + self.game_install_progress = builder.get_object('game_install_progress') + self.global_progression = builder.get_object('global_progression') + + self.game_install_timeout_id = None + self.current_selected_game = None + self.game_image_pixbuf = None + + def on_destroy(self, *args): + gtk.main_quit() + + def on_quit_clicked(self, *args): + print('quit') + gtk.main_quit() + + def gamelist_on_changed(self, selection, *args): + model, treeiter = selection.get_selected() + if treeiter is None: + return + index = model[treeiter][2] + #print("You selected", self.gamelist['games'][model[treeiter][2]]) + debug('selected index: %d', index) + game_item = self.gamelist.yaml_data['games'][index] + self.current_selected_game = game_item + info("You selected (title): %s", game_item['title']) + + self.game_title.set_label(game_item['title']) + if image := game_item.get('image', None): + image_path = f'{self.gamelist_dir}/{image}' + debug('image path: %s', image_path) + if os.path.exists(image_path): + #self.game_image.set_from_file(image_path) + self.game_image_pixbuf = Pixbuf.new_from_file(image_path) + self.game_image_draw() + self.game_characteristics.set_text( + game_item.get('characteristics', 'Unfilled characteristics')) + description = game_item.get('description', 'No description') + self.game_description.set_text(description) + + if self.game_status.exists(game_item['name']): + debug('game status exists') + self.btn_install.hide() + self.btn_update.show() + self.btn_remove.show() + else: + debug('game status does not exist') + + def game_image_on_draw(self, widget, cr, *args): + self.game_image_draw() + + def game_image_draw(self): + if not self.game_image_pixbuf: + return + + orig_width = self.game_image_pixbuf.get_width() + orig_height = self.game_image_pixbuf.get_height() + dest_width = self.game_image.get_allocated_width() + dest_height = self.game_image.get_allocated_height() + ratio_width = dest_width / orig_width + ratio_height = dest_height / orig_height + ratio = min(ratio_width, ratio_height) + + #width, height = self.game_image.get_allocated_size() + pixbuftmp = self.game_image_pixbuf.scale_simple( + orig_width*ratio, + orig_height*ratio, + gdkpixbuf.InterpType.BILINEAR) + self.game_image.set_from_pixbuf(pixbuftmp) + + def on_draw(self, widget, cr, data): + context = widget.get_style_context() + + width = widget.get_allocated_width() + height = widget.get_allocated_height() + Gtk.render_background(context, cr, 0, 0, width, height) + + r, g, b, a = data["color"] + cr.set_source_rgba(r, g, b, a) + cr.rectangle(0, 0, width, height) + cr.fill() + + ############################################################ + # install management + ############################################################ + + def game_install_on_clicked(self, *args): + if not self.current_selected_game: + return + + self.game_install_timeout_id = \ + glib.timeout_add(250, self.game_install_on_timeout, None) + self.game_install_stop_btn.show() + + game_item = self.current_selected_game + self.game_installer.start(game_item) + # set progress bar to 0.0001 in order to update them the first time on + # timeout (most useful for global advance) + self.global_progression.set_fraction(0.0001) + self.game_install_progress.set_fraction(0.0001) + + def game_install_stop(self): + if not self.game_install_timeout_id: + return + glib.source_remove(self.game_install_timeout_id) + self.game_install_timeout_id = None + self.game_installer.stop() + #self.game_install_progress.set_fraction(1) + self.game_install_stop_btn.hide() + + def game_installstop_on_clicked(self, *args): + self.game_install_stop() + + def game_install_on_timeout(self, *args): + status, progress1, progress2 = self.game_installer.poll() + #print(f'status {status} progress1 {progress1} progress2 {progress2}') + old_progress1 = self.global_progression.get_fraction() + if (progress1/100) != old_progress1: + self.global_progression.set_fraction(progress1/100) + self.global_progression.set_text(f'global {progress1:.2f}%') + old_progress2 = self.game_install_progress.get_fraction() + if (progress2/100) != old_progress2: + self.game_install_progress.set_fraction(progress2/100) + self.game_install_progress.set_text(f'step {progress2:.2f}%') + # returning False value means stop timer + if status == self.game_installer.State.ONGOING: + return True + self.game_install_stop() + return False + + +def main(): + logging.basicConfig(level=logging.DEBUG) + logging.getLogger().handlers[0].setFormatter(logging.Formatter( + "gamechest: %(levelname)s: %(funcName)s:%(lineno)s: %(message)s")) + + args = docopt.docopt(__doc__) + debug('args: %s', args) + + gamelist_dir = ( + os.path.dirname(args['--gamelist']) + + '/.' + + os.path.basename(args['--gamelist']) + + '.d' + ) + gamelist = GameList(args['--gamelist'], args['--gamedir']) + + builder = gtk.Builder() + builder.add_from_file('gamechest.glade') + builder.connect_signals( + Handler(builder, gamelist, gamelist_dir, args['--gamedir'])) + + window = builder.get_object('window2') + window.show_all() + + gamelist_store = builder.get_object('gamelist_store') + gamelist_store.clear() + for index, game in enumerate(gamelist.yaml_data['games']): + gamelist_store.append((False, game['title'], index)) + info('%d games loaded', len(gamelist.yaml_data['games'])) + + gtk.main() diff --git a/gamechest/cli.py b/gamechest/cli.py new file mode 100644 index 0000000..d0b457e --- /dev/null +++ b/gamechest/cli.py @@ -0,0 +1,66 @@ +#!python3 + +''' +Usage: gamechest.py --gamelist=GAMELIST.YAML --gamedir=GAMEDIR +''' + +import logging +import os +from logging import debug, info, warning, critical + + +import docopt +import yaml +import gamemanager + + +def main(): + logging.basicConfig(level=logging.DEBUG) + logging.getLogger().handlers[0].setFormatter(logging.Formatter( + "gamechest: %(levelname)s: %(funcName)s:%(lineno)s: %(message)s")) + + args = docopt.docopt(__doc__) + debug('args: %s', args) + + gamelist_dir = ( + os.path.dirname(args['--gamelist']) + + '/.' + + os.path.basename(args['--gamelist']) + + '.d' + ) + gamelist = GameList(args['--gamelist'], args['--gamedir']) + + builder = gtk.Builder() + builder.add_from_file('gamechest.glade') + builder.connect_signals( + Handler(builder, gamelist, gamelist_dir, args['--gamedir'])) + + window = builder.get_object('window2') + window.show_all() + + gamelist_store = builder.get_object('gamelist_store') + gamelist_store.clear() + for index, game in enumerate(gamelist.yaml_data['games']): + gamelist_store.append((False, game['title'], index)) + info('%d games loaded', len(gamelist.yaml_data['games'])) + + gtk.main() + +def main(): + import sys + gamelist_path = sys.argv[1] + gamedir_path = sys.argv[2] + action_name = sys.argv[3] + game_name = sys.argv[4] # used to determine game_id + + # TODO: + # - parse gamelist (conf) + # - parse package installed info or empty (conf) + # - follow action (install, remove, update) + # - use name on game name + # - update status bar from polling (with a timer, or a simple sleep) + # - if run, run the game. + + +if __name__ == '__main__': + main() diff --git a/gamechest/conf.py b/gamechest/conf.py new file mode 100644 index 0000000..17e7737 --- /dev/null +++ b/gamechest/conf.py @@ -0,0 +1,25 @@ +#!python3 + +import yaml + +class Config: + ''' + Stores configuration for whole program run time. + + Should be pretty unmuttable. + ''' + + def __init__(self, gamelist_path, gamedir): + self.gamelist_path = gamelist_path + self.gamedir = gamedir + + # parse gamelist.yaml file; parsed once at init time and never written + # (not meat to be writtable). + with open(gamelist_path, 'r', encoding='utf8') as stream: + self.gamelist = yaml.safe_load(stream) + + self.repository_host = self.gamelist['repository']['host'] + self.repository_path = self.gamelist['repository']['path'] + + def get_status(self, game_id): + 'get the status of a game by its game_id' diff --git a/gamechest/gamemanager.py b/gamechest/gamemanager.py new file mode 100644 index 0000000..2e9b06c --- /dev/null +++ b/gamechest/gamemanager.py @@ -0,0 +1,183 @@ +#!python3 + +import contextlib +from logging import debug, info, warning, critical +from enum import Enum, auto as enum_auto + +import yaml + +#from . import utils +from stepper import Stepper +import steps_install, steps_remove + +def strip_tar_ext(name): + ext = '.tar.lzip' + return name[:-len(ext)] if name.lower().endswith(ext) else name + + + +_installer_steps_dict = { + 'download': steps_install.step_rsync, + 'extract': steps_install.step_extract, + 'clean': steps_install.step_clean, +} +_remover_steps_dict = { + 'remove': steps_remove.step_remove, +} +_updater_steps_dict = dict(**_remover_steps_dict, **_installer_steps_dict) + + +class GameStepper: + + def __init__(self, conf): + self.game_installer = Stepper(conf, installer_steps_dict) + self.game_remover = Stepper(conf, remover_steps_dict) + self.game_updater = Stepper(conf, updater_steps_dict) + self.curent = None + + def install(self, game_id): + if self.current: + return + self.current = self.game_installer + self.game_installer.start(self.conf.get_status(game_id)) + + def remove(self, game_id): + if self.current: + return + self.current = self.game_remover + self.game_remover.start(self.conf.get_status(game_id)) + + def update(self, game_id): + if self.current: + return + self.current = self.game_updater + self.game_updater.start(self.conf.get_status(game_id)) + + def stop(self): + if self.current: + self.current.stop() + self.current = None + + def poll(self): + if self.current: + return self.current.poll() + + +class GameManager: + + class State(Enum): + IDLE = enum_auto() + ONGOING = enum_auto() + + def __init__(self, conf): + self.game_installer = Stepper(conf, installer_steps_dict) + self.game_remover = Stepper(conf, remover_steps_dict) + self.game_updater = Stepper(conf, updater_steps_dict) + + #def __init__(self, games_data_yaml_path, games_dir): + # self.games_data_yaml_path = games_data_yaml_path + # self.games_dir = games_dir + # self.state = self.State.IDLE + # self.update_state = self.State.IDLE + # self.current_game_name = None + # self.current_game_item = None + # self.current_game_status = {} + + # with open(games_data_yaml_path) as stream: + # self.games_data = yaml.safe_load(stream) + + # self.game_installer = GameInstaller( + # self.games_data['repository']['host'], + # self.games_data['repository']['path'], + # self.games_dir, + # ) + # self.game_remover = GameRemover( + # self.games_data['repository']['host'], + # self.games_data['repository']['path'], + # self.games_dir, + # ) + + def get_game_item(self, name): + self.current_game_item = next( + item + for item in self.games_data['games'] + if item['name'] == name + ) + return self.current_game_item + + def get_game_status(self, name): + self.current_game_status = {} + with contextlib.suppress(FileNotFoundError): + with open(f'{self.conf.gamedir}/{name}.yaml') as stream: + self.current_game_status = yaml.safe_load(stream) + return self.current_game_status + + def is_game_updatable(self, name): + #self. ### FIXME: Stopped here ? + if not self.current_game_name: + return False + ivers = self.current_game_status.get('installed_version', None) + # only call "updatable" if there was at least a version installed + # else, this is just an "installable" package. + if ivers is not None and ivers < self.current_game_item['version']: + return True + return False + + def set_current_game(self, game_id): + if self.current_game_name == name: + return + self.current_game_name = name + self.get_game_item(name) + self.get_game_status(name) + + def start_install(self, game_id): + if self.state != self.State.IDLE: + return + self.status = self.State.INSTALLING + self.set_current_game(name) + self.game_installer.start( + self.current_game_item, + game_status=self.current_game_status, + ) + + def start_remove(self, name): + if self.state != self.State.IDLE: + return + self.status = self.State.REMOVING + self.set_current_game(name) + self.game_remover.start( + self.current_game_item, + game_status=self.current_game_status, + ) + + def start_update(self, name): + if self.state != self.State.IDLE: + return + self.status = self.State.UPDATING_REMOVING + self.set_current_game(name) + self.game_remover.start( + self.current_game_item, + game_status=self.current_game_status, + ) + + def stop(self): + self.state = self.State.IDLE + self.game_remover.stop() + self.game_installer.stop() + + def poll(self): + entry_state = self.state + return_value = { + self.State.IDLE: (lambda: GameManagerBase.State.IDLE, 0, 0), + self.State.INSTALLING: self.game_installer.poll, + self.State.REMOVING: self.game_remover.poll, + self.State.UPDATING_REMOVING: self.game_remover.poll, + self.State.UPDATING_INSTALLING: self.game_installer.poll, + }[entry_state]() + if return_value[0] == GameManagerBase.Sate.IDLE: + self.state = self.State.IDLE + if entry_state == self.State.UPDATING_REMOVING: + self.start_install(self.current_game_name) + return_value = self.poll() + return self.state, return_value[1], return_value[2] + diff --git a/gamechest/gamemanager.py.backup b/gamechest/gamemanager.py.backup new file mode 100644 index 0000000..f8f9551 --- /dev/null +++ b/gamechest/gamemanager.py.backup @@ -0,0 +1,431 @@ +#!python3 + +import contextlib +import os +import queue +import re +import selectors +import shutil +import subprocess +import tarfile +import threading +from logging import debug, info, warning, critical +from enum import Enum, auto as enum_auto + +import yaml + + +def strip_tar_ext(name): + ext = '.tar.lzip' + return name[:-len(ext)] if name.lower().endswith(ext) else name + + +class GameChestConfig: + ''' + Stores configuration for whole program run time. + + Should be pretty unmuttable. + ''' + + def __init__(self, repository_host, repository_path, gamedir, game_data): + self.repository_host = repository_host + self.repository_path = repository_path + self.gamedir = gamedir + self.game_data = game_data + + + +class LockedData: + 'every attribute access to this class is locked except lock and data' + + def __init__(self, data={}): + super().__setattr__('lock', threading.Lock()) + super().__setattr__('data', dict()) + self.data.update(data) + + def __getattr__(self, name): + with self.lock: + return self.data[name] + + def __setattr__(self, name, value): + with self.lock: + self.data[name] = value + + +class Progress(LockedData): + 'hold a progression status' + + def __init__(self): + super().__init__(self, { + 'global': 0, + 'steps_count': 1, + 'step': 0, + 'step_index': 0, + 'step_name': 'unknown', + }) + + @locked + def set_step_index(self, index): + self._step_index = index + self._global = (index/self._steps_count)*100 + + +class GameManagerBase: + ''' + Manage a worker processing steps in background and allows polling its + status (global steps progress, and current step progress). + ''' + + class State(Enum): + IDLE = enum_auto() + ONGOING = enum_auto() + + class Command(Enum): + STOP = enum_auto() + + def __init__(self, conf): + self.conf = conf + self.state_holder = LockedData() + self.state = self.State.IDLE + self.thread = None + self.lock = threading.Lock() + self.queue = queue.Queue(maxsize=10) + self.progress = Progress() + + @property + def state(self): + return self.state_holder.state + + @property.setter + def set_state(self, value): + self.state_holder.state = value + + def worker(self, *args): + self.progress.steps_count = len(self.steps) + try: + for index, step in enumerate(self.steps): + self.progress.step_index = index + for step_progress in step(): + self.progress.step = step_progress + finally: + self.state = self.State.IDLE + + def start(self, *, game_status=None): + if self.steps is None: + raise NotImplementedError + if self.state == self.State.ONGOING: + return + self.progress.global = 0 + self.progress.step = 0 + with self.lock: + self.game_status = game_status + self.thread = threading.Thread(target=self.worker, daemon=True) + self.thread.start() + self.state = self.State.ONGOING + + def stop(self): + if self.state != self.State.ONGOING: + return + self.state = self.State.IDLE + self.queue.put(self.Command.STOP) + + def poll(self): + return self.state, self.global_progress, self.step_progress + + +class GameInstaller(GameManagerBase): + ''' + Defines how to install a game: + - download it + - unarchive it + - remove the archive + ''' + + rsync_progress_re = re.compile(r'\s(\d+)%\s') + + def __init__(self, conf): + super().__init__(conf) + self.steps = ( + self._worker_rsync, + self._worker_unarchive, + self._worker_rmarchive, + ) + + def _worker_rsync_loop(self, proc, selector): + while True: + with contextlib.suppress(queue.Empty): + if self.queue.get_nowait() == self.Command.STOP: + proc.terminate() + break + + if proc.poll() is not None: + break + + if not any(key + for key, mask in selector.select(timeout=0.250) + if mask & selectors.EVENT_READ and key.fileobj == proc.stdout): + continue + + # stdout.readline is a blocking call if there is no endline in + # stdout outputed by the subprocess. + # normally rsync output should be line buffered thus at any read event + # a call to readline should not block (or not long enough to be + # able to process a queued command quick enough). + line = proc.stdout.readline() + if match := self.rsync_progress_re.search(line): + progress = int(match.group(1)) + yield progress + + def _worker_rsync(self): + package_name = self.conf.game_data['package_name'] + command = ( + 'rsync', + '-a', + '--partial', + '--info=progress2', + f'{self.conf.repository_host}:{self.conf.repository_path}/{package_name}', + f'{self.conf.gamedir}/.', + ) + debug('running command %s', command) + with contextlib.ExitStack() as stack: + proc = stack.enter_context(subprocess.Popen( + command, + stdout=subprocess.PIPE, + encoding='utf8')) + selector = stack.enter_context(selectors.DefaultSelector()) + selector.register(proc.stdout, selectors.EVENT_READ) + yield from self._worker_rsync_loop(proc=proc, selector=selector) + + def _worker_unarchive(self): + with contextlib.ExitStack() as stack: + lzip_command = '/usr/bin/lzip' + plzip_command = '/usr/bin/plzip' + if os.path.exists(plzip_command): + lzip_command = plzip_command + lzip_proc = stack.enter_context(subprocess.Popen( + (lzip_command, '-d'), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + )) + tar_proc = stack.enter_context(subprocess.Popen( + ('tar', '-C', self.output_dir, '-xvf', '-'), + stdin=lzip_proc.stdout, + stdout=subprocess.PIPE, + )) + filepath = f'{self.conf.gamedir}/{self.conf.game_data["package_name"]}' + filepath_status = f'{self.conf.gamedir}/{self.conf.game_data["name"]}.yaml' + fobj = stack.enter_context(open(filepath, 'rb')) + selector = stack.enter_context(selectors.DefaultSelector()) + selector.register(tar_proc.stdout, selectors.EVENT_READ) + filesize = os.stat(filepath).st_size + def _itemscounter(): + while ready := selector.select(0.100): + data = tar_proc.stdout.read1() + if len(data) == 0: + break + itemscount += data.count(b'\n') + itemscount = 0 + while True: + with contextlib.suppress(queue.Empty): + if self.queue.get_nowait() == self.Command.STOP: + break + data = fobj.read(131072) # 128kib chunks + if len(data) == 0: + lzip_proc.stdin.close() + break + lzip_proc.stdin.write(data) + yield filesize / fobj.tell() + _itemscounter() + _itemscounter() + with open(filepath_status, 'wb') as fobj: + yaml.safe_dump({ + 'installed_count': itemscount, + 'installed_version': self.conf.game_data['version'], + 'installed_package_name': self.conf.game_data['package_name'], + }, fobj) + + def _worker_rmarchive(self): + os.unlink(f'{self.conf.gamedir}/{self.conf.game_data["package_name"]}') + + +class GameRemover(GameManagerBase): + ''' + Defines how to remove a game. Basically removes a directory, with progress + report. + ''' + + def __init__(self, conf): + super().__init__(conf) + self.steps = (self._worker_remove, self._worker_rm_data) + + def _worker_remove(self): + if self.game_status is None: + return + extracted_path = '/'.join(( + self.conf.gamedir, + strip_tar_ext(self.game_status['installed_package_name']), + )) + filepath_status = f'{self.conf.gamedir}/{self.conf.game_data["name"]}.yaml' + with open(filepath_status) as stream: + pkg_status = yaml.safe_load(stream) + with contextlib.ExitStack() as stack: + proc = stack.enter_context(subprocess.Popen( + ('rm', '-rvf', '--', extracted_path), + stdout=subprocess.PIPE, + )) + selector = stack.enter_context(selectors.DefaultSelector()) + selector.register(proc.stdout, selectors.EVENT_READ) + itemscount = 0 + while True: + with contextlib.suppress(queue.Empty): + if self.queue.get_nowait() == self.Command.STOP: + proc.terminate() + break + if ready := selector.select(0.250): + data = proc.stdout.read1() + if len(data) == 0: + break + itemscount += data.count(b'\n') + yield itemscount*100/pkg_status['installed_count'] + + def _worker_rm_data(self): + if self.game_status is None: + return + os.unlink(f'{self.conf.gamedir}/{self.conf.game_data["name"]}.yaml') + + +class GameUpdater(GameManagerBase): + ''' + Combine GameRemover and GameInstaller to form a consistent updating unit. + ''' + + def __init__(self, conf): + super().__init__(conf) + self._remover = GameRemover(conf) + self._installer = GameInstaller(conf) + self.steps = self._remover.steps + self._installer.steps + + + def _worker(self): + + +#class GameData: +# +# class + + +class GameManager: + + class State(Enum): + IDLE = enum_auto() + INSTALLING = enum_auto() + REMOVING = enum_auto() + UPDATING_REMOVING = enum_auto() + UPDATING_INSTALLING = enum_auto() + + def __init__(self, games_data_yaml_path, games_dir): + self.games_data_yaml_path = games_data_yaml_path + self.games_dir = games_dir + self.state = self.State.IDLE + self.update_state = self.State.IDLE + self.current_game_name = None + self.current_game_item = None + self.current_game_status = {} + + with open(games_data_yaml_path) as stream: + self.games_data = yaml.safe_load(stream) + + self.game_installer = GameInstaller( + self.games_data['repository']['host'], + self.games_data['repository']['path'], + self.games_dir, + ) + self.game_remover = GameRemover( + self.games_data['repository']['host'], + self.games_data['repository']['path'], + self.games_dir, + ) + + def get_game_item(self, name): + self.current_game_item = next( + item + for item in self.games_data['games'] + if item['name'] == name + ) + return self.current_game_item + + def get_game_status(self, name): + self.current_game_status = {} + with contextlib.suppress(FileNotFoundError): + with open(f'{self.conf.gamedir}/{name}.yaml') as stream: + self.current_game_status = yaml.safe_load(stream) + return self.current_game_status + + def is_game_updatable(self, name): + #self. ### FIXME: Stopped here ? + if not self.current_game_name: + return False + ivers = self.current_game_status.get('installed_version', None) + # only call "updatable" if there was at least a version installed + # else, this is just an "installable" package. + if ivers is not None and ivers < self.current_game_item['version']: + return True + return False + + def set_current_game(self, name): + if self.current_game_name == name: + return + self.current_game_name = name + self.get_game_item(name) + self.get_game_status(name) + + def start_install(self, name): + if self.state != self.State.IDLE: + return + self.status = self.State.INSTALLING + self.set_current_game(name) + self.game_installer.start( + self.current_game_item, + game_status=self.current_game_status, + ) + + def start_remove(self, name): + if self.state != self.State.IDLE: + return + self.status = self.State.REMOVING + self.set_current_game(name) + self.game_remover.start( + self.current_game_item, + game_status=self.current_game_status, + ) + + def start_update(self, name): + if self.state != self.State.IDLE: + return + self.status = self.State.UPDATING_REMOVING + self.set_current_game(name) + self.game_remover.start( + self.current_game_item, + game_status=self.current_game_status, + ) + + def stop(self): + self.state = self.State.IDLE + self.game_remover.stop() + self.game_installer.stop() + + def poll(self): + entry_state = self.state + return_value = { + self.State.IDLE: (lambda: GameManagerBase.State.IDLE, 0, 0), + self.State.INSTALLING: self.game_installer.poll, + self.State.REMOVING: self.game_remover.poll, + self.State.UPDATING_REMOVING: self.game_remover.poll, + self.State.UPDATING_INSTALLING: self.game_installer.poll, + }[entry_state]() + if return_value[0] == GameManagerBase.Sate.IDLE: + self.state = self.State.IDLE + if entry_state == self.State.UPDATING_REMOVING: + self.start_install(self.current_game_name) + return_value = self.poll() + return self.state, return_value[1], return_value[2] diff --git a/gamechest/gamemanager_method1.py b/gamechest/gamemanager_method1.py new file mode 100644 index 0000000..9e2d4ac --- /dev/null +++ b/gamechest/gamemanager_method1.py @@ -0,0 +1,252 @@ +#!python3 + +import contextlib +import os +import queue +import re +import selectors +import shutil +import subprocess +import tarfile +import threading +from logging import debug, info, warning, critical +from enum import Enum, auto as enum_auto + + +class TarLzipExtractor: + + class State(Enum): + IDLE = enum_auto() + ONGOING = enum_auto() + + class Command(Enum): + STOP = enum_auto() + + def __init__(self, output_dir): + self.output_dir = output_dir + self.state_lock = threading.Lock() + self.state = self.State.IDLE + self.queue = queue.Queue(maxsize=10) + self.progress_lock = threading.Lock() + self.progress = 0 + + def _thread_worker(self, *args): + lzip_proc = tar_proc = None + try: + lzip_command = '/usr/bin/lzip' + plzip_command = '/usr/bin/plzip' + if os.path.exists(plzip_command): + lzip_command = plzip_command + lzip_proc = subprocess.Popen( + (lzip_command, '-d'), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + tar_proc = subprocess.Popen( + ('tar', '-C', self.output_dir, '-xf', '-'), + stdin=lzip_proc.stdout, + ) + filepath = f'{self.output_dir}/{self.filename}' + filesize = os.stat(filepath).st_size + progress = 0 + with open(filepath, 'rb') as fobj: + while True: + with contextlib.suppress(queue.Empty): + queue_command = self.queue.get_nowait() + if queue_command == self.Command.STOP: + return + data = fobj.read(131072) # 128kib chunks + if len(data) == 0: + lzip_proc.stdin.close() + return + lzip_proc.stdin.write(data) + with self.progress_lock: + self.progress = filesize / fobj.tell() + finally: + with self.state_lock: + self.state = self.State.IDLE + # first terminate all, then wait all, quicker than terminate/wait + # for each. + lzip_proc or lzip_proc.terminate() + tar_proc or tar_proc.terminate() + lzip_proc or lzip_proc.wait() + tar_proc or tar_proc.wait() + + def start(self, filename): + + with self.state_lock: + if self.state == self.State.ONGOING: + return + self.state = self.State.ONGOING + + self.filename = filename + + self.thread = threading.Thread( + target=self._thread_worker, + daemon=True) + self.thread.start() + + def stop(self): + + with self.state_lock: + if self.state != self.State.ONGOING: + return + self.state = self.State.IDLE + + self.queue.put(self.Command.STOP) + + def poll(self): + with self.state_lock, self.progress_lock: + return self.state, self.progress + + +class RsyncManager: + + rsync_progress_re = re.compile(r'\s(\d+)%\s') + + class State(Enum): + IDLE = enum_auto() + ONGOING = enum_auto() + + def __init__(self, repository_host, repository_path, gamedir): + self.selector = selectors.DefaultSelector() + self.repository_host = repository_host + self.repository_path = repository_path + self.gamedir = gamedir + + self.state = self.State.IDLE + self.proc = None + self.package_name = None + self.last_progress = 0 + + def start(self, package_name): + 'returns True when package_name is being installed' + + if self.state == self.State.ONGOING: + return + + self.package_name = package_name + command = ( + 'rsync', + '-a', + '--partial', + '--info=progress2', + f'{self.repository_host}:{self.repository_path}/{package_name}', + f'{self.gamedir}/.', + ) + debug('running command %s', command) + self.proc = subprocess.Popen( + command, + stdout=subprocess.PIPE, + encoding='utf8') + self.selector.register(self.proc.stdout, selectors.EVENT_READ) + self.last_progress = 0 + self.state = self.State.ONGOING + + return True + + def stop(self): + if self.state == self.State.IDLE: + return + if self.proc.poll() is None: + debug('terminating unfinished game install process for %s', + self.package_name) + self.proc.terminate() + proc_return_code = self.proc.wait() + debug('game install process return status %d', proc_return_code) + self.selector.unregister(self.proc.stdout) + self.proc = None + self.install_ongoing = False + + def poll(self): + if self.state == self.State.IDLE: + return self.state, self.last_progress + + proc = self.proc + if proc.poll() is not None: + self.stop() + self.state = self.State.IDLE + return self.state, 100 + + if not any(key + for key, mask in self.selector.select(timeout=0) + if mask & selectors.EVENT_READ and key.fileobj == proc.stdout): + return self.state, self.last_progress + + # stdout.readline is a blocking call if there is no endline in + # stdout outputed by the subprocess. + # normally rsync output should be line buffered thus at any read event + # a call to readline should not block (or not long enough to be + # visible in a gui). + line = proc.stdout.readline() + if match := self.rsync_progress_re.search(line): + progress = int(match.group(1)) + self.last_progress = progress + + return self.state, self.last_progress + + +class GameArchiveRemover: + + def __init__(self, gamedir): + self.gamedir = gamedir + + def start(self, package_name): + pass + + def stop(self): + pass + + def poll(self): + pass + + +class GameInstaller: + + class State(Enum): + IDLE = enum_auto() + ONGOING = enum_auto() + + def __init__(self, repository_host, repository_path, gamedir): + self.state = self.State.IDLE + self.step = None + self.steps = ( + RsyncManager( + repository_host=repository_host, + repository_path=repository_path, + gamedir=gamedir + ), + TarLzipExtractor(output_dir=gamedir), + ) + + def start(self, package_name): + if self.state == self.State.ONGOING: + return + self.step = 0 + self.steps[self.step].start(package_name) + self.package_name = package_name + self.state = self.State.ONGOING + #self.rsync_manager.start(package_name) + #self.current_sub_process = self.rsync_manager + return True + + def stop(self): + if self.state == self.State.IDLE: + return + self.steps[self.step].stop() + #self.current_sub_process.stop() + self.state = self.State.IDLE + + def poll(self): + 'update internal state and returns current state' + if self.state == self.State.IDLE: + return self.State.IDLE, 100, 100 + sub_state, sub_progress = self.steps[self.step].poll() + if sub_state == self.steps[self.step].State.IDLE: + self.step += 1 + if self.step >= len(self.steps): + self.state = self.State.IDLE + return self.State.IDLE, 100, 100 + self.steps[self.step].start(self.package_name) + sub_progress = 0 + return self.state, (self.step/len(self.steps))*100, sub_progress diff --git a/gamechest/stepper.py b/gamechest/stepper.py new file mode 100644 index 0000000..b6dd335 --- /dev/null +++ b/gamechest/stepper.py @@ -0,0 +1,69 @@ +#!python3 + +import collections +import copy +from enum import Enum, auto as enum_auto + +Progress = collections.namedtuple('Progress', 'step index count percent') +Progress.step.__doc__ = 'name of the current step' +Progress.index.__doc__ = 'current step index out of count steps' +Progress.count.__doc__ = 'total number of steps' +Progress.percent.__doc__ = 'advance in percent of the current step' + +class Stepper: + ''' + Manage a worker processing steps in background and allows polling its + progress. + + Takes steps as a dict of step_names/step_worker. + A step_worker is a function generator taking a queue waiting for a stop + command to abort the step and yielding progress of the step as + a percentage, and as second argument it takes a GameChestConfig. + ''' + + class Command(Enum): + STOP = enum_auto() + + def _create_thread(self): + return threading.Thread(target=self.worker, daemon=True) + + def __init__(self, conf, steps): + self.conf = conf + self.thread = None + self.commandq = queue.Queue(maxsize=10) + self.progressq = queue.Queue(maxsize=10) + self.steps = steps + self.count = len(steps) + self.steps.__doc__ = 'steps are a key/func-value dict' + self.thread = self._create_thread() + self.last_progress = Progress('unknown', 0, self.count, 0) + self.status = {} + + def worker(self, *args): + for index, step, step_worker in enumerate(self.steps.items()): + self.progressq.put(Progress(step, index, self.count, 0)) + for percent in step_worker(self.commandq, self.conf, self.status): + self.progressq.put(Progress(step, index, self.count, percent)) + + def start(self, status): + if self.is_alive(): + return + self.status = copy.deepcopy(status) + self.thread.start() + + def stop(self): + if not self.is_alive(): + return + self.commandq.put(self.Command.STOP) + self.thread.join() + self.thread = self._create_thread() + + def is_alive(self): + return self.thread.is_alive() + + def poll(self): + with contextlib.suppress(queue.Empty): + progress = self.progress_queue.get_nowait() + self.last_progress = progress + return self.last_progress + diff --git a/gamechest/steps_install.py b/gamechest/steps_install.py new file mode 100644 index 0000000..7c508bb --- /dev/null +++ b/gamechest/steps_install.py @@ -0,0 +1,109 @@ +#!python3 + +import contextlib +import os +import re +import selectors +import subprocess + +from stepper import Stepper + +_rsync_progress_re = re.compile(r'\s(\d+)%\s') + +def _rsync_loop(queue, proc, selector): + while True: + with contextlib.suppress(queue.Empty): + if queue.get_nowait() == Stepper.Command.STOP: + proc.terminate() + break + + if proc.poll() is not None: + break + + if not any(key + for key, mask in selector.select(timeout=0.250) + if mask & selectors.EVENT_READ and key.fileobj == proc.stdout): + continue + + # stdout.readline is a blocking call if there is no endline in + # stdout outputed by the subprocess. + # normally rsync output should be line buffered thus at any read event + # a call to readline should not block (or not long enough to be + # able to process a queued command quick enough). + line = proc.stdout.readline() + if match := _rsync_progress_re.search(line): + progress = int(match.group(1)) + yield progress + +def step_rsync(queue, conf, status): + package_name = conf.game_data['package_name'] + command = ( + 'rsync', + '-a', + '--partial', + '--info=progress2', + f'{conf.repository_host}:{conf.repository_path}/{package_name}', + f'{conf.gamedir}/.', + ) + debug('running command %s', command) + with contextlib.ExitStack() as stack: + proc = stack.enter_context(subprocess.Popen( + command, + stdout=subprocess.PIPE, + encoding='utf8')) + selector = stack.enter_context(selectors.DefaultSelector()) + selector.register(proc.stdout, selectors.EVENT_READ) + yield from _rsync_loop(queue, proc=proc, selector=selector) + +def step_extract(queue, conf, status): + with contextlib.ExitStack() as stack: + lzip_command = '/usr/bin/lzip' + plzip_command = '/usr/bin/plzip' + if os.path.exists(plzip_command): + lzip_command = plzip_command + lzip_proc = stack.enter_context(subprocess.Popen( + (lzip_command, '-d'), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + )) + tar_proc = stack.enter_context(subprocess.Popen( + ('tar', '-C', conf.gamedir, '-xvf', '-'), + stdin=lzip_proc.stdout, + stdout=subprocess.PIPE, + )) + filepath = f'{conf.gamedir}/{conf.game_data["package_name"]}' + filepath_status = f'{conf.gamedir}/{conf.game_data["name"]}.yaml' + fobj = stack.enter_context(open(filepath, 'rb')) + selector = stack.enter_context(selectors.DefaultSelector()) + selector.register(tar_proc.stdout, selectors.EVENT_READ) + filesize = os.stat(filepath).st_size + def _itemscounter(): + while ready := selector.select(0.100): + data = tar_proc.stdout.read1() + if len(data) == 0: + break + itemscount += data.count(b'\n') + itemscount = 0 + while True: + with contextlib.suppress(queue.Empty): + if queue.get_nowait() == Stepper.Command.STOP: + break + data = fobj.read(131072) # 128kib chunks + if len(data) == 0: + lzip_proc.stdin.close() + break + lzip_proc.stdin.write(data) + yield filesize / fobj.tell() + _itemscounter() + _itemscounter() + with open(filepath_status, 'wb') as fobj: + yaml.safe_dump({ + 'installed_count': itemscount, + 'installed_version': conf.game_data['version'], + 'installed_package_name': conf.game_data['package_name'], + }, fobj) + +def step_clean(queue, conf, status): + os.unlink(f'{conf.gamedir}/{conf.game_data["package_name"]}') + yield 100 + diff --git a/gamechest/steps_remove.py b/gamechest/steps_remove.py new file mode 100644 index 0000000..6b9144a --- /dev/null +++ b/gamechest/steps_remove.py @@ -0,0 +1,31 @@ +#!python3 + +import contextlib +import subprocess + +from stepper import Stepper + +def step_remove(queue, conf, status): + extracted_path = '/'.join(( + conf.gamedir, + strip_tar_ext(status['installed_package_name']), + )) + with contextlib.ExitStack() as stack: + proc = stack.enter_context(subprocess.Popen( + ('rm', '-rvf', '--', extracted_path), + stdout=subprocess.PIPE, + )) + selector = stack.enter_context(selectors.DefaultSelector()) + selector.register(proc.stdout, selectors.EVENT_READ) + itemscount = 0 + while True: + with contextlib.suppress(queue.Empty): + if queue.get_nowait() == Stepper.Command.STOP: + proc.terminate() + break + if ready := selector.select(0.250): + data = proc.stdout.read1() + if len(data) == 0: + break + itemscount += data.count(b'\n') + yield itemscount*100/status['installed_count'] diff --git a/gamechest/utils.py b/gamechest/utils.py new file mode 100644 index 0000000..adc55e7 --- /dev/null +++ b/gamechest/utils.py @@ -0,0 +1,19 @@ +#!python3 + +import threading + +class LockedData: + 'every attribute access to this class is locked except lock and data' + + def __init__(self, data={}): + super().__setattr__('lock', threading.Lock()) + super().__setattr__('data', dict()) + self.data.update(data) + + def __getattr__(self, name): + with self.lock: + return self.data[name] + + def __setattr__(self, name, value): + with self.lock: + self.data[name] = value -- cgit v1.2.3