1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
|
#!python3
import contextlib
import os
import re
import selectors
import subprocess
from stepper import Stepper
_rsync_progress_re = re.compile(r'\s(\d+)%\s')
def _rsync_loop(queue, proc, selector):
while True:
with contextlib.suppress(queue.Empty):
if queue.get_nowait() == Stepper.Command.STOP:
proc.terminate()
break
if proc.poll() is not None:
break
if not any(key
for key, mask in selector.select(timeout=0.250)
if mask & selectors.EVENT_READ and key.fileobj == proc.stdout):
continue
# stdout.readline is a blocking call if there is no endline in
# stdout outputed by the subprocess.
# normally rsync output should be line buffered thus at any read event
# a call to readline should not block (or not long enough to be
# able to process a queued command quick enough).
line = proc.stdout.readline()
if match := _rsync_progress_re.search(line):
progress = int(match.group(1))
yield progress
def step_rsync(queue, conf, status):
package_name = conf.game_data['package_name']
command = (
'rsync',
'-a',
'--partial',
'--info=progress2',
f'{conf.repository_host}:{conf.repository_path}/{package_name}',
f'{conf.gamedir}/.',
)
debug('running command %s', command)
with contextlib.ExitStack() as stack:
proc = stack.enter_context(subprocess.Popen(
command,
stdout=subprocess.PIPE,
encoding='utf8'))
selector = stack.enter_context(selectors.DefaultSelector())
selector.register(proc.stdout, selectors.EVENT_READ)
yield from _rsync_loop(queue, proc=proc, selector=selector)
def step_extract(queue, conf, status):
with contextlib.ExitStack() as stack:
lzip_command = '/usr/bin/lzip'
plzip_command = '/usr/bin/plzip'
if os.path.exists(plzip_command):
lzip_command = plzip_command
lzip_proc = stack.enter_context(subprocess.Popen(
(lzip_command, '-d'),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
))
tar_proc = stack.enter_context(subprocess.Popen(
('tar', '-C', conf.gamedir, '-xvf', '-'),
stdin=lzip_proc.stdout,
stdout=subprocess.PIPE,
))
filepath = f'{conf.gamedir}/{conf.game_data["package_name"]}'
filepath_status = f'{conf.gamedir}/{conf.game_data["name"]}.yaml'
fobj = stack.enter_context(open(filepath, 'rb'))
selector = stack.enter_context(selectors.DefaultSelector())
selector.register(tar_proc.stdout, selectors.EVENT_READ)
filesize = os.stat(filepath).st_size
def _itemscounter():
while ready := selector.select(0.100):
data = tar_proc.stdout.read1()
if len(data) == 0:
break
itemscount += data.count(b'\n')
itemscount = 0
while True:
with contextlib.suppress(queue.Empty):
if queue.get_nowait() == Stepper.Command.STOP:
break
data = fobj.read(131072) # 128kib chunks
if len(data) == 0:
lzip_proc.stdin.close()
break
lzip_proc.stdin.write(data)
yield filesize / fobj.tell()
_itemscounter()
_itemscounter()
with open(filepath_status, 'wb') as fobj:
yaml.safe_dump({
'installed_count': itemscount,
'installed_version': conf.game_data['version'],
'installed_package_name': conf.game_data['package_name'],
}, fobj)
def step_clean(queue, conf, status):
os.unlink(f'{conf.gamedir}/{conf.game_data["package_name"]}')
yield 100
|
