Why does QtNetwork.QFtp.get download fail in a for-loop for multiple files?

Question

I am trying to download multiple files from a ftp site using a for-loop. The following code seems to work only for the first 2 files in the loop before the python.exe shutdown window pops up. Two downloaded files are perfect, but the 3rd downloaded file is empty upon the shutdown. I don't get the rest of files. Any idea what could be the problem?

from PyQt4 import QtCore, QtGui, QtNetwork


class FtpWindow(QtGui.QDialog):

    def __init__(self, parent=None):
        self.fileList = QtGui.QTreeWidget()
        self.ftp = QtNetwork.QFtp(self)
        self.progressDialog = QtGui.QProgressDialog(self)
        self.downloadAllButton.clicked.connect(self.downloadAllFile)
        self.ftp.commandFinished.connect(self.ftpCommandFinished)

    def downloadAllFile(self):
        for jj in range(self.fileList.topLevelItemCount()): # how many files in a particular folder
            fileName = self.fileList.topLevelItem(jj).text(0) 
            self.outFile = QtCore.QFile(fileName)

            self.ftp.get(fileName, self.outFile) #download one file at a time
            self.progressDialog.setLabelText("Downloading %s..." % fileName)    
            self.progressDialog.exec_()

    def ftpCommandFinished(self, _, error):
        self.setCursor(QtCore.Qt.ArrowCursor)
        if self.ftp.currentCommand() == QtNetwork.QFtp.Get:
            if error:
                self.statusLabel.setText("Canceled download of %s." % self.outFile.fileName())
                self.outFile.close()
                self.outFile.remove()
            else:
                self.statusLabel.setText("Downloaded %s to current directory." % self.outFile.fileName())
                self.outFile.close()

            self.outFile = None
            self.enableDownloadButton()
            self.progressDialog.hide()

Answer 1

Thanks for HashSplat's input. I have a few updates to make it fully functional:

class FtpWindow(QtGui.QDialog):

def __init__(self, parent=None):
    self.fileList = QtGui.QTreeWidget()
    self.ftp = QtNetwork.QFtp(self)
    self.progressDialog = QtGui.QProgressDialog(self)
    self.progressDialog.canceled.connect(self.ftp.abort)
    self.downloadAllButton.clicked.connect(self.downloadAllFile)
    self.ref_holder = {}
    self.ftp.commandFinished.connect(self.ftpCommandFinished)

def download_file(self, fileName):
    """Non blocking start downloading a file."""
    self.outFile = QtCore.QFile(fileName)

    """ Need this to have files saved locally """
    if not self.outFile.open(QtCore.QIODevice.WriteOnly):  
        QtGui.QMessageBox.information(self, "FTP",
                "Unable to save the file %s." % fileName)
        self.outFile = None
        return
    cmd_id = self.ftp.get(filename, self.outFile) # Non blocking just start downloading

    # This keeps the object alive and doesn't overwrite them.
    self.ref_holder[cmd_id] = [filename, self.outFile]

def downloadAllFile(self):
    self.progressDialog.reset()
    self.num_downloads = self.fileList.topLevelItemCount()
    self.counter=1
    self.progressDialog.setLabelText("Downloading %d/%d files ..." % (self.counter, self.num_downloads))
    self.progressDialog.show()
    for jj in range(num_downloads): # how many files in a particular folder
        fileName = self.fileList.topLevelItem(jj).text(0) 
        self.download_file(fileName) # Non blocking, and doesn't overwrite self.outFile with every iteration

def ftpCommandFinished(self, cmd_id, error=None):
    """Increased the number of items finished."""
    self.progressDialog.setValue(self.progressDialog.value()+1)
    item = self.ref_holder.pop(cmd_id) # Remove the reference for the finished item
    if error:
        self.progressDialog.setLabelText("Error downloading %s" % item[0])

    # Check if all downloads are done
    if len(self.ref_holder) == 0:
        self.progressDialog.close() # This closes the extra window
        self.outFile.close()   # You need this to have the last file saved
    else: 
        self.counter+=1
        self.progressDialog.setLabelText("Downloading %d/%d files ..." % (self.counter, self.num_downloads))

def updateDataTransferProgress(self, readBytes, totalBytes):
    self.progressDialog.setMaximum(totalBytes)
    self.progressDialog.setValue(readBytes)

Answer 2

self.progressDialog.exec_() should be a blocking modal dialog. Use self.progressDialog.show() for a non blocking call.

It looks like the ftp get is non blocking so you have to wait until downloading is finished using the commandFinished() signal.

My guess is that every iteration in the loop is overwriting the self.outFile, so there isn't any python references to the object. This makes the object die whenever python does garbage collection. My guess is that your first two files where small and quick and your third file was larger, so the other files were able to download before garbage collection. Either that or garbage collection was just quicker for the last file.

http://pyside.github.io/docs/pyside/PySide/QtNetwork/QFtp.html#PySide.QtNetwork.PySide.QtNetwork.QFtp.get

class FtpWindow(QtGui.QDialog):

    def __init__(self, parent=None):
        self.fileList = QtGui.QTreeWidget()
        self.ftp = QtNetwork.QFtp(self)
        self.progressDialog = QtGui.QProgressDialog(self)
        self.progressDialog.canceled.connect(self.ftp.abort)
        self.downloadAllButton.clicked.connect(self.downloadAllFile)
        self.ref_holder = {}
        self.ftp.commandFinished.connect(self.ftpCommandFinished)

    def download_file(self, filename):
        """Non blocking start downloading a file."""
        outFile = QtCore.QFile(filename)
        cmd_id = self.ftp.get(filename, outFile) # Non blocking just start downloading

        # This keeps the object alive and doesn't overwrite them.
        self.ref_holder[cmd_id] = [filename, outFile]

    def downloadAllFile(self):
        self.progressDialog.reset()
        num_downloads = self.fileList.topLevelItemCount()
        self.progressDialog.setMaximum(num_downloads)
        self.progressDialog.setValue(0)
        self.progressDialog.setLabelText("Downloading %d files ..." % num_downloads)
        self.progressDialog.show()
        for jj in range(num_downloads): # how many files in a particular folder
            fileName = self.fileList.topLevelItem(jj).text(0) 
            self.download_file(fileName) # Non blocking, and doesn't overwrite self.outFile with every iteration

    def ftpCommandFinished(self, cmd_id, error=None):
        """Increased the number of items finished."""
        self.progressDialog.setValue(self.progressDialog.value()+1)
        item = self.ref_holder.pop(cmd_id) # Remove the reference for the finished item
        if error:
            self.progressDialog.setLabelText("Error downloading %s" % item[0])

        # Check if all downloads are done
        if len(self.ref_holder) == 0:
            self.progressDialog.setValue(self.progressDialog.maximium())
            self.progressDialog.close() # This shouldn't be needed

My example above will hold the filename and outFile object reference until the command is finished. When the command is finished the reference is removed allowing python to clean up the object.