^thehatsrule^
Group: Members
Posts: 3275
Joined: July 2006 |
 |
Posted: June 23 2008,04:58 |
 |
This is forked off from this thread. The recent events reminded me of lua... and I took the suggestion from MaxiJavi in that thread and decided to take a look at it - and then I ended up modifying it. I added resuming downloads and to use the basename of the url as the filename by default if arg2 is not specified (I messed up my terminal many times since the original prints to stdout by default). This code is not robust - it just mostly shows that it can work (hopefully). It didn't take a lot of time, but I did get hung up on the ftp resume part - "fixed" after some trial and error. Does anyone know if there's a real fix to this, or if this is a bug in the platform?
The original can be found in [luasocket]/etc/get.lua. I've only tested this on local httpd/ftpd's and a bit on the DSL download mirrors. But I haven't tried it on the lua environment on the latest 4.4.x yet.
Going through this made me think that the use of ltn12 is pretty neat.
get2-003.lua:
| Code Sample | -----------------------------------------------------------------------------
-- Little program to download files from URLs
-- LuaSocket sample files
-- Author: Diego Nehab
-- Extended to add resume by ^thehatsrule^
-- RCS ID: $Id: get.lua,v 1.25 2007/03/12 04:08:40 diego Exp $
-----------------------------------------------------------------------------
local socket = require("socket")
local http = require("socket.http")
local ftp = require("socket.ftp")
local url = require("socket.url")
local ltn12 = require("ltn12")
local lfs = require("lfs")
-- formats a number of seconds into human readable form
function nicetime(s)
local l = "s"
if s > 60 then
s = s / 60
l = "m"
if s > 60 then
s = s / 60
l = "h"
if s > 24 then
s = s / 24
l = "d" -- hmmm
end
end
end
if l == "s" then return string.format("%5.0f%s", s, l)
else return string.format("%5.2f%s", s, l) end
end
-- formats a number of bytes into human readable form
function nicesize(b)
local l = "B"
if b > 1024 then
b = b / 1024
l = "KB"
if b > 1024 then
b = b / 1024
l = "MB"
if b > 1024 then
b = b / 1024
l = "GB" -- hmmm
end
end
end
return string.format("%7.2f%2s", b, l)
end
-- returns a string with the current state of the download
local remaining_s = "%s received, %s/s throughput, %2.0f%% done, %s remaining"
local elapsed_s = "%s received, %s/s throughput, %s elapsed "
function gauge(got, delta, size)
local rate = got / delta
if size and size >= 1 then
return string.format(remaining_s, nicesize(got), nicesize(rate),
100*got/size, nicetime((size-got)/rate))
else
return string.format(elapsed_s, nicesize(got),
nicesize(rate), nicetime(delta))
end
end
-- creates a new instance of a receive_cb that saves to disk
-- kind of copied from luasocket's manual callback examples
function stats(size)
local start = socket.gettime()
local last = start
local got = 0
return function(chunk)
-- elapsed time since start
local current = socket.gettime()
if chunk then
-- total bytes received
got = got + string.len(chunk)
-- not enough time for estimate
if current - last > 1 then
io.stderr:write("\r", gauge(got, current - start, size))
io.stderr:flush()
last = current
end
else
-- close up
io.stderr:write("\r", gauge(got, current - start), "\n")
end
return chunk
end
end
-- determines the size of a http file
function gethttpsize(u)
local r, c, h = http.request {method = "HEAD", url = u}
if c == 200 then
return tonumber(h["content-length"])
end
end
-- downloads a file using the http protocol
function getbyhttp(u, file, sz)
local save = ltn12.sink.file(file or io.stdout)
local size2get = gethttpsize(u)
if not size2get then
io.stderr:write("Error contacting remote host.")
--if not sz then do_something() end
os.exit(0)
end
if size2get == sz then
print("File already completed.")
os.exit(0)
end
-- only print feedback if output is not stdout
if file then save = ltn12.sink.chain(stats(size2get), save) end
local hdrs
if sz then
-- resume header
hdrs = { ["Range"] = "bytes=" .. sz .."-" }
end
local r, c, h, s = http.request {url = u, sink = save, headers = hdrs }
-- todo: check for 206; fallback?
if c ~= 200 then io.stderr:write(s or c, "\n") end
end
-- downloads a file using the ftp protocol
function getbyftp(u, file, sz)
local save = ltn12.sink.file(file or io.stdout)
-- only print feedback if output is not stdout
-- and we don't know how big the file is
-- todo: can use SIZE file to get file size, but is not RFC
-- would probably be easier to do 2 ftp sessions
if file then save = ltn12.sink.chain(stats(), save) end
local gett = url.parse(u)
gett.sink = save
gett.type = "i"
if sz then
-- try to resume; fallback?
-- filler command (see below) + restart offset + retrieve file
-- bug? having REST first will return 350, ftp.get returns early
gett.command = 'SYST' .. '\n' .. 'REST ' .. sz .. '\nRETR'
end
local ret, err = ftp.get(gett)
if err then print(err) end
end
-- determines the scheme
function getscheme(u)
-- this is an heuristic to solve a common invalid url poblem
if not string.find(u, "//") then u = "//" .. u end
local parsed = url.parse(u, {scheme = "http"})
return parsed.scheme
end
-- gets a file either by http or ftp, saving as <name>
-- todo: check filesize etc before opening file
function get(u, name)
if not name then
-- use the basename from the url, assuming no spaces
-- better to exec `basename` instead?
name = string.gsub(u, '.*/', "")
-- todo: change this
if not name then
name = os.tmpname()
print ("Using " .. name .. " to store data")
end
end
local sz = lfs.attributes(name, "size")
local mode
if sz then
print ("File " .. name .. " found with a size of " .. sz)
print ("Trying to resume ...")
mode = "a+b"
else
mode = "wb"
end
local fout = name and io.open(name, mode)
local scheme = getscheme(u)
if scheme == "ftp" then getbyftp(u, fout, sz)
elseif scheme == "http" then getbyhttp(u, fout, sz)
else print("unknown scheme" .. scheme) end
end
-- main program
arg = arg or {}
if table.getn(arg) < 1 then
io.write("Usage:\n lua get.lua <remote-url> [<local-file>]\n")
os.exit(1)
else get(arg[1], arg[2]) end |
|
.
DSL Market ,
Great VPS hosting provided by Tektonic
