1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
# -*- coding: utf-8 -*-
from ftplib import FTP
import datetime, functools, os, time, tempfile
from source import Installer
import utils
def _timestampFromFilename( platform, fn ):
# windows: assumed format is YYYY_mm_dd_HH_MM as prefix
# linux: assumed format is YYYY-mm-dd as suffix
if platform.startswith( 'windows' ):
format = '%Y_%m_%d_%H_%M'
length = len( 'YYYY_mm_dd_HH_MM' )
return datetime.datetime.strptime( fn[0:length], format )
else:
format = '%Y-%m-%d'
length = len( 'YYYY-mm-dd' )
return datetime.datetime.strptime( fn[-length:], format )
def timestampFromFilename( platform, fn ):
try:
return _timestampFromFilename( platform, fn )
except ValueError:
return None
def filesNewerThan( files, dt ):
if dt:
return filter( lambda (x,y): y >= dt, files )
else:
return files
#internal to FtpSource
class Location:
def __init__( self, host, path, platform ):
self.host = host
self.path = path
self.platform = platform
self.testedFiles = []
def ls( self ):
ftp = FTP( self.host )
ftp.login()
try:
return map( utils.basename, ftp.nlst( self.path ) )
finally:
ftp.close()
def filesSortedByTimestamp( self ):
files = self.ls()
withTS = [( i, timestampFromFilename( self.platform, i ) ) for i in files]
filtered = filter( lambda (x,y): y != None, withTS )
filtered.sort( key=lambda (x,y): y )
return filtered
def untestedFilesSortedByTimestamp( self ):
l = self.filesSortedByTimestamp()
return filter( lambda ( x, y ): not self.isTested( x ), l )
def markAsTested( self, filename ):
self.testedFiles.append( filename )
def isTested( self, filename ):
return filename in self.testedFiles
def description( self ):
return "host={0} path={1} platform={2}".format( self.host, self.path, self.platform )
def download( self, fn, target ):
ftp = FTP( self.host )
ftp.login()
try:
ftp.retrbinary( 'RETR {0}/{1}'.format( self.path, fn ), target.write )
finally:
ftp.close()
class FtpSource:
def __init__( self, delay=60*60, tempdir='/tmp' ):
self._locations = []
self._delay = delay
self._tempdir = tempdir
self._startDate = None
def setStartDate( self, date ):
self._startDate = date
def addLocation( self, host, path, platform ):
self._locations.append( Location( host, path, platform ) )
def nextInstaller( self ):
while True:
for i in self._locations:
print( "** Checking FTP location: " + i.description() )
files = i.untestedFilesSortedByTimestamp()
if self._startDate != None:
files = filesNewerThan( files, self._startDate )
if len( files ) == 0:
continue;
fn, ts = files[0]
print( "** Downloading new installer: {0}...".format( fn ) )
tmp = tempfile.NamedTemporaryFile( dir=self._tempdir, prefix=fn )
i.download( fn, tmp )
print( "** Download completed. ({0})".format( tmp.name ) )
i.lastTested = ts
inst = Installer( tmp.name, i.platform, i, ts, tmp )
inst.sourceFilename = fn
return inst
print( "** No installers found. Going to sleep for {0} seconds...".format( self._delay ) )
time.sleep( self._delay )
if __name__ == "__main__":
src = FtpSource()
src.addLocation( "hegel", "/projects/ndk/installers/windows", "windows" )
while True:
inst = src.nextInstaller()
print inst
inst.markAsTested()
|
