from __future__ import with_statement
__license__   = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'

# Imports {{{
import os, traceback, Queue, time, cStringIO, re, sys, weakref
from threading import Thread, Event

from PyQt4.Qt import (QMenu, QAction, QActionGroup, QIcon, SIGNAL,
                     Qt, pyqtSignal, QDialog, QObject, QVBoxLayout,
                     QDialogButtonBox)

from calibre.customize.ui import (available_input_formats, available_output_formats,
    device_plugins, disabled_device_plugins)
from calibre.devices.interface import DevicePlugin
from calibre.devices.errors import (UserFeedback, OpenFeedback, OpenFailed,
                                    InitialConnectionError)
from calibre.gui2.dialogs.choose_format_device import ChooseFormatDeviceDialog
from calibre.utils.ipc.job import BaseJob
from calibre.devices.scanner import DeviceScanner
from calibre.gui2 import (config, error_dialog, Dispatcher, dynamic,
        warning_dialog, info_dialog, choose_dir, FunctionDispatcher,
        show_restart_warning, gprefs, question_dialog)
from calibre.ebooks.metadata import authors_to_string
from calibre import preferred_encoding, prints, force_unicode, as_unicode
from calibre.utils.filenames import ascii_filename
from calibre.devices.errors import (FreeSpaceError, WrongDestinationError,
        BlacklistedDevice)
from calibre.devices.apple.driver import ITUNES_ASYNC
from calibre.devices.folder_device.driver import FOLDER_DEVICE
from calibre.devices.bambook.driver import BAMBOOK, BAMBOOKWifi
from calibre.constants import DEBUG
from calibre.utils.config import tweaks, device_prefs
from calibre.utils.magick.draw import thumbnail
from calibre.library.save_to_disk import find_plugboard
# }}}

class DeviceJob(BaseJob): # {{{

    def __init__(self, func, done, job_manager, args=[], kwargs={},
            description=''):
        BaseJob.__init__(self, description)
        self.func = func
        self.callback_on_done = done
        if not isinstance(self.callback_on_done, (Dispatcher,
            FunctionDispatcher)):
            self.callback_on_done = FunctionDispatcher(self.callback_on_done)

        self.args, self.kwargs = args, kwargs
        self.exception = None
        self.job_manager = job_manager
        self._details = _('No details available.')
        self._aborted = False

    def start_work(self):
        if DEBUG:
            prints('Job:', self.id, self.description, 'started',
                safe_encode=True)
        self.start_time = time.time()
        self.job_manager.changed_queue.put(self)

    def job_done(self):
        self.duration = time.time() - self.start_time
        self.percent = 1
        if DEBUG:
            prints('DeviceJob:', self.id, self.description,
                    'done, calling callback', safe_encode=True)

        try:
            self.callback_on_done(self)
        except:
            pass
        if DEBUG:
            prints('DeviceJob:', self.id, self.description,
                    'callback returned', safe_encode=True)
        self.job_manager.changed_queue.put(self)

    def report_progress(self, percent, msg=''):
        self.notifications.put((percent, msg))
        self.job_manager.changed_queue.put(self)

    def run(self):
        self.start_work()
        try:
            self.result = self.func(*self.args, **self.kwargs)
            if self._aborted:
                return
        except (Exception, SystemExit) as err:
            if self._aborted:
                return
            self.failed = True
            ex = as_unicode(err)
            self._details = ex + '\n\n' + \
                force_unicode(traceback.format_exc())
            self.exception = err
        finally:
            self.job_done()

    def abort(self, err):
        call_job_done = False
        if self.run_state == self.WAITING:
            self.start_work()
            call_job_done = True
        self._aborted = True
        self.failed = True
        self._details = unicode(err)
        self.exception = err
        if call_job_done:
            self.job_done()

    @property
    def log_file(self):
        return cStringIO.StringIO(self._details.encode('utf-8'))

    # }}}

def device_name_for_plugboards(device_class):
    if hasattr(device_class, 'DEVICE_PLUGBOARD_NAME'):
        return device_class.DEVICE_PLUGBOARD_NAME
    return device_class.__class__.__name__

class DeviceManager(Thread): # {{{

    def __init__(self, connected_slot, job_manager, open_feedback_slot,
            open_feedback_msg, allow_connect_slot, sleep_time=2):
        '''
        :sleep_time: Time to sleep between device probes in secs
        '''
        Thread.__init__(self)
        self.setDaemon(True)
        # [Device driver, Showing in GUI, Ejected]
        self.devices        = list(device_plugins())
        self.disabled_device_plugins = list(disabled_device_plugins())
        self.managed_devices = [x for x in self.devices if
                not x.MANAGES_DEVICE_PRESENCE]
        self.unmanaged_devices = [x for x in self.devices if
                x.MANAGES_DEVICE_PRESENCE]
        self.sleep_time     = sleep_time
        self.connected_slot = connected_slot
        self.allow_connect_slot = allow_connect_slot
        self.jobs           = Queue.Queue(0)
        self.job_steps      = Queue.Queue(0)
        self.keep_going     = True
        self.job_manager    = job_manager
        self.reported_errors = set([])
        self.current_job    = None
        self.scanner        = DeviceScanner()
        self.connected_device = None
        self.connected_device_kind = None
        self.ejected_devices  = set([])
        self.mount_connection_requests = Queue.Queue(0)
        self.open_feedback_slot = open_feedback_slot
        self.open_feedback_msg = open_feedback_msg
        self._device_information = None
        self.current_library_uuid = None
        self.call_shutdown_on_disconnect = False
        self.devices_initialized = Event()
        self.dynamic_plugins = {}

    def report_progress(self, *args):
        pass

    @property
    def is_device_connected(self):
        return self.connected_device is not None

    @property
    def device(self):
        return self.connected_device

    def do_connect(self, connected_devices, device_kind):
        for dev, detected_device in connected_devices:
            if dev.OPEN_FEEDBACK_MESSAGE is not None:
                self.open_feedback_slot(dev.OPEN_FEEDBACK_MESSAGE)
            try:
                dev.reset(detected_device=detected_device,
                    report_progress=self.report_progress)
                dev.open(detected_device, self.current_library_uuid)
            except OpenFeedback as e:
                if dev not in self.ejected_devices:
                    self.open_feedback_msg(dev.get_gui_name(), e)
                    self.ejected_devices.add(dev)
                continue
            except OpenFailed:
                raise
            except:
                tb = traceback.format_exc()
                if DEBUG or tb not in self.reported_errors:
                    self.reported_errors.add(tb)
                    prints('Unable to open device', str(dev))
                    prints(tb)
                continue
            self.after_device_connect(dev, device_kind)
            return True
        return False

    def after_device_connect(self, dev, device_kind):
        allow_connect = True
        try:
            uid = dev.get_device_uid()
        except NotImplementedError:
            uid = None
        asked = gprefs.get('ask_to_manage_device', [])
        if (dev.ASK_TO_ALLOW_CONNECT and uid and uid not in asked):
            if not self.allow_connect_slot(dev.get_gui_name(), dev.icon):
                allow_connect = False
            asked.append(uid)
            gprefs.set('ask_to_manage_device', asked)
        if not allow_connect:
            dev.ignore_connected_device(uid)
            return

        self.connected_device = dev
        self.connected_device.specialize_global_preferences(device_prefs)
        self.connected_device_kind = device_kind
        self.connected_slot(True, device_kind)

    def connected_device_removed(self):
        while True:
            try:
                job = self.jobs.get_nowait()
                job.abort(Exception(_('Device no longer connected.')))
            except Queue.Empty:
                break
        try:
            self.connected_device.post_yank_cleanup()
        except:
            pass
        if self.connected_device in self.ejected_devices:
            self.ejected_devices.remove(self.connected_device)
        else:
            self.connected_slot(False, self.connected_device_kind)
        if self.call_shutdown_on_disconnect:
            # The current device is an instance of a plugin class instantiated
            # to handle this connection, probably as a mounted device. We are
            # now abandoning the instance that we created, so we tell it that it
            # is being shut down.
            self.connected_device.shutdown()
            self.call_shutdown_on_disconnect = False
        device_prefs.set_overrides()
        self.connected_device = None
        self._device_information = None

    def detect_device(self):
        self.scanner.scan()

        if self.is_device_connected:
            if self.connected_device.MANAGES_DEVICE_PRESENCE:
                cd = self.connected_device.detect_managed_devices(self.scanner.devices)
                if cd is None:
                    self.connected_device_removed()
            else:
                connected, detected_device = \
                    self.scanner.is_device_connected(self.connected_device,
                            only_presence=True)
                if not connected:
                    if DEBUG:
                        # Allow the device subsystem to output debugging info about
                        # why it thinks the device is not connected. Used, for e.g.
                        # in the can_handle() method of the T1 driver
                        self.scanner.is_device_connected(self.connected_device,
                                only_presence=True, debug=True)
                    self.connected_device_removed()
        else:
            for dev in self.unmanaged_devices:
                try:
                    cd = dev.detect_managed_devices(self.scanner.devices)
                except:
                    prints('Error during device detection for %s:'%dev)
                    traceback.print_exc()
                else:
                    if cd is not None:
                        try:
                            dev.open(cd, self.current_library_uuid)
                        except BlacklistedDevice as e:
                            prints('Ignoring blacklisted device: %s'%
                                    as_unicode(e))
                        except:
                            prints('Error while trying to open %s (Driver: %s)'%
                                    (cd, dev))
                            traceback.print_exc()
                        else:
                            self.after_device_connect(dev, 'unmanaged-device')
                            return
            try:
                possibly_connected_devices = []
                for device in self.managed_devices:
                    if device in self.ejected_devices:
                        continue
                    try:
                        possibly_connected, detected_device = \
                                self.scanner.is_device_connected(device)
                    except InitialConnectionError as e:
                        self.open_feedback_msg(device.get_gui_name(), e)
                        continue
                    if possibly_connected:
                        possibly_connected_devices.append((device, detected_device))
                if possibly_connected_devices:
                    if not self.do_connect(possibly_connected_devices,
                                           device_kind='device'):
                        if DEBUG:
                            prints('Connect to device failed, retrying in 5 seconds...')
                        time.sleep(5)
                        if not self.do_connect(possibly_connected_devices,
                                           device_kind='device'):
                            if DEBUG:
                                prints('Device connect failed again, giving up')
            except OpenFailed as e:
                if e.show_me:
                    traceback.print_exc()

    # Mount devices that don't use USB, such as the folder device and iTunes
    # This will be called on the GUI thread. Because of this, we must store
    # information that the scanner thread will use to do the real work.
    def mount_device(self, kls, kind, path):
        self.mount_connection_requests.put((kls, kind, path))

    # disconnect a device
    def umount_device(self, *args):
        if self.is_device_connected and not self.job_manager.has_device_jobs():
            if self.connected_device_kind in {'unmanaged-device', 'device'}:
                self.connected_device.eject()
                if self.connected_device_kind != 'unmanaged-device':
                    self.ejected_devices.add(self.connected_device)
                self.connected_slot(False, self.connected_device_kind)
            elif hasattr(self.connected_device, 'unmount_device'):
                # As we are on the wrong thread, this call must *not* do
                # anything besides set a flag that the right thread will see.
                self.connected_device.unmount_device()

    def next(self):
        if not self.job_steps.empty():
            try:
                return self.job_steps.get_nowait()
            except Queue.Empty:
                pass

        if not self.jobs.empty():
            try:
                return self.jobs.get_nowait()
            except Queue.Empty:
                pass

    def run_startup(self, dev):
        name = 'unknown'
        try:
            name = dev.__class__.__name__
            dev.startup()
        except:
            prints('Startup method for device %s threw exception'%name)
            traceback.print_exc()

    def run(self):
        # Do any device-specific startup processing.
        for d in self.devices:
            self.run_startup(d)
            n = d.is_dynamically_controllable()
            if n:
                self.dynamic_plugins[n] = d
        self.devices_initialized.set()

        while self.keep_going:
            kls = None
            while True:
                try:
                    (kls,device_kind, folder_path) = \
                                self.mount_connection_requests.get_nowait()
                except Queue.Empty:
                    break
            if kls is not None:
                try:
                    dev = kls(folder_path)
                    # We just created a new device instance. Call its startup
                    # method and set the flag to call the shutdown method when
                    # it disconnects.
                    self.run_startup(dev)
                    self.call_shutdown_on_disconnect = True
                    self.do_connect([[dev, None],], device_kind=device_kind)
                except:
                    prints('Unable to open %s as device (%s)'%(device_kind, folder_path))
                    traceback.print_exc()
            else:
                self.detect_device()

            do_sleep = True
            while True:
                job = self.next()
                if job is not None:
                    do_sleep = False
                    self.current_job = job
                    if self.device is not None:
                        self.device.set_progress_reporter(job.report_progress)
                    self.current_job.run()
                    self.current_job = None
                else:
                    break
            if do_sleep:
                time.sleep(self.sleep_time)

        # We are exiting. Call the shutdown method for each plugin
        for p in self.devices:
            try:
                p.shutdown()
            except:
                pass

    def create_job_step(self, func, done, description, to_job, args=[], kwargs={}):
        job = DeviceJob(func, done, self.job_manager,
                        args=args, kwargs=kwargs, description=description)
        self.job_manager.add_job(job)
        if (done is None or isinstance(done, FunctionDispatcher)) and \
                    (to_job is not None and to_job == self.current_job):
            self.job_steps.put(job)
        else:
            self.jobs.put(job)
        return job

    def create_job(self, func, done, description, args=[], kwargs={}):
        return self.create_job_step(func, done, description, None, args, kwargs)

    def has_card(self):
        try:
            return bool(self.device.card_prefix())
        except:
            return False

    def _debug_detection(self):
        from calibre.devices import debug
        raw = debug(plugins=self.devices,
                disabled_plugins=self.disabled_device_plugins)
        return raw

    def debug_detection(self, done):
        if self.is_device_connected:
            raise ValueError('Device is currently detected in calibre, cannot'
                    ' debug device detection')
        self.create_job(self._debug_detection, done,
                _('Debug device detection'))

    def _get_device_information(self):
        info = self.device.get_device_information(end_session=False)
        if len(info) < 5:
            info = tuple(list(info) + [{}])
        info = [i.replace('\x00', '').replace('\x01', '') if isinstance(i, basestring) else i
                 for i in info]
        cp = self.device.card_prefix(end_session=False)
        fs = self.device.free_space()
        self._device_information = {'info': info, 'prefixes': cp, 'freespace': fs}
        return info, cp, fs

    def get_device_information(self, done, add_as_step_to_job=None):
        '''Get device information and free space on device'''
        return self.create_job_step(self._get_device_information, done,
                    description=_('Get device information'), to_job=add_as_step_to_job)

    def slow_driveinfo(self):
        ''' Update the stored device information with the driveinfo if the
        device indicates that getting driveinfo is slow '''
        info = self._device_information['info']
        if (not info[4] and self.device.SLOW_DRIVEINFO):
            info = list(info)
            info[4] = self.device.get_driveinfo()
            self._device_information['info'] = tuple(info)

    def get_current_device_information(self):
        return self._device_information

    def _books(self):
        '''Get metadata from device'''
        mainlist = self.device.books(oncard=None, end_session=False)
        cardalist = self.device.books(oncard='carda')
        cardblist = self.device.books(oncard='cardb')
        return (mainlist, cardalist, cardblist)

    def books(self, done, add_as_step_to_job=None):
        '''Return callable that returns the list of books on device as two booklists'''
        return self.create_job_step(self._books, done,
                description=_('Get list of books on device'), to_job=add_as_step_to_job)

    def _prepare_addable_books(self, paths):
        return self.device.prepare_addable_books(paths)

    def prepare_addable_books(self, done, paths, add_as_step_to_job=None):
        return self.create_job_step(self._prepare_addable_books, done, args=[paths],
                description=_('Prepare files for transfer from device'),
                to_job=add_as_step_to_job)

    def _annotations(self, path_map):
        return self.device.get_annotations(path_map)

    def annotations(self, done, path_map, add_as_step_to_job=None):
        '''Return mapping of ids to annotations. Each annotation is of the
        form (type, location_info, content). path_map is a mapping of
        ids to paths on the device.'''
        return self.create_job_step(self._annotations, done, args=[path_map],
                description=_('Get annotations from device'), to_job=add_as_step_to_job)

    def _sync_booklists(self, booklists):
        '''Sync metadata to device'''
        self.device.sync_booklists(booklists, end_session=False)
        return self.device.card_prefix(end_session=False), self.device.free_space()

    def sync_booklists(self, done, booklists, plugboards, add_as_step_to_job=None):
        if hasattr(self.connected_device, 'set_plugboards') and \
                callable(self.connected_device.set_plugboards):
            self.connected_device.set_plugboards(plugboards, find_plugboard)
        return self.create_job_step(self._sync_booklists, done, args=[booklists],
                        description=_('Send metadata to device'), to_job=add_as_step_to_job)

    def upload_collections(self, done, booklist, on_card, add_as_step_to_job=None):
        return self.create_job_step(booklist.rebuild_collections, done,
                               args=[booklist, on_card],
                        description=_('Send collections to device'),
                        to_job=add_as_step_to_job)

    def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None):
        '''Upload books to device: '''
        from calibre.ebooks.metadata.meta import set_metadata
        if hasattr(self.connected_device, 'set_plugboards') and \
                callable(self.connected_device.set_plugboards):
            self.connected_device.set_plugboards(plugboards, find_plugboard)
        if metadata and files and len(metadata) == len(files):
            for f, mi in zip(files, metadata):
                if isinstance(f, unicode):
                    ext = f.rpartition('.')[-1].lower()
                    cpb = find_plugboard(
                            device_name_for_plugboards(self.connected_device),
                            ext, plugboards)
                    if ext:
                        try:
                            if DEBUG:
                                prints('Setting metadata in:', mi.title, 'at:',
                                        f, file=sys.__stdout__)
                            with open(f, 'r+b') as stream:
                                if cpb:
                                    newmi = mi.deepcopy_metadata()
                                    newmi.template_to_attribute(mi, cpb)
                                else:
                                    newmi = mi
                                nuke_comments = getattr(self.connected_device,
                                        'NUKE_COMMENTS', None)
                                if nuke_comments is not None:
                                    mi.comments = nuke_comments
                                set_metadata(stream, newmi, stream_type=ext)
                        except:
                            if DEBUG:
                                prints(traceback.format_exc(), file=sys.__stdout__)

        try:
            return self.device.upload_books(files, names, on_card,
                    metadata=metadata, end_session=False)
        finally:
            if metadata:
                for mi in metadata:
                    try:
                        if mi.cover:
                            os.remove(mi.cover)
                    except:
                        pass

    def upload_books(self, done, files, names, on_card=None, titles=None,
                     metadata=None, plugboards=None, add_as_step_to_job=None):
        desc = _('Upload %d books to device')%len(names)
        if titles:
            desc += u':' + u', '.join(titles)
        return self.create_job_step(self._upload_books, done, to_job=add_as_step_to_job,
                               args=[files, names],
                kwargs={'on_card':on_card,'metadata':metadata,'plugboards':plugboards}, description=desc)

    def add_books_to_metadata(self, locations, metadata, booklists):
        self.device.add_books_to_metadata(locations, metadata, booklists)

    def _delete_books(self, paths):
        '''Remove books from device'''
        self.device.delete_books(paths, end_session=True)

    def delete_books(self, done, paths, add_as_step_to_job=None):
        return self.create_job_step(self._delete_books, done, args=[paths],
                        description=_('Delete books from device'),
                        to_job=add_as_step_to_job)

    def remove_books_from_metadata(self, paths, booklists):
        self.device.remove_books_from_metadata(paths, booklists)

    def _save_books(self, paths, target):
        '''Copy books from device to disk'''
        for path in paths:
            name = path.rpartition(os.sep)[2]
            dest = os.path.join(target, name)
            if os.path.abspath(dest) != os.path.abspath(path):
                f = open(dest, 'wb')
                self.device.get_file(path, f)
                f.close()

    def save_books(self, done, paths, target, add_as_step_to_job=None):
        return self.create_job_step(self._save_books, done, args=[paths, target],
                        description=_('Download books from device'),
                        to_job=add_as_step_to_job)

    def _view_book(self, path, target):
        with open(target, 'wb') as f:
            self.device.get_file(path, f)
        return target

    def view_book(self, done, path, target, add_as_step_to_job=None):
        return self.create_job_step(self._view_book, done, args=[path, target],
                        description=_('View book on device'), to_job=add_as_step_to_job)

    def set_current_library_uuid(self, uuid):
        self.current_library_uuid = uuid

    def set_driveinfo_name(self, location_code, name):
        if self.connected_device:
            self.connected_device.set_driveinfo_name(location_code, name)

    # dynamic plugin interface

    # This is a helper function that handles queueing with the device manager
    def _call_request(self, name, method, *args, **kwargs):
        d = self.dynamic_plugins.get(name, None)
        if d:
            return getattr(d, method)(*args, **kwargs)
        return kwargs.get('default', None)

    # The dynamic plugin methods below must be called on the GUI thread. They
    # will switch to the device thread before calling the plugin.

    def start_plugin(self, name):
        return self._call_request(name, 'start_plugin')

    def stop_plugin(self, name):
        self._call_request(name, 'stop_plugin')

    def get_option(self, name, opt_string, default=None):
        return self._call_request(name, 'get_option', opt_string, default=default)

    def set_option(self, name, opt_string, opt_value):
        self._call_request(name, 'set_option', opt_string, opt_value)

    def is_running(self, name):
        if self._call_request(name, 'is_running'):
            return True
        return False

    def is_enabled(self, name):
        try:
            d = self.dynamic_plugins.get(name, None)
            if d:
                return True
        except:
            pass
        return False

    # }}}

class DeviceAction(QAction): # {{{

    a_s = pyqtSignal(object)

    def __init__(self, dest, delete, specific, icon_path, text, parent=None):
        QAction.__init__(self, QIcon(icon_path), text, parent)
        self.dest = dest
        self.delete = delete
        self.specific = specific
        self.triggered.connect(self.emit_triggered)

    def emit_triggered(self, *args):
        self.a_s.emit(self)

    def __repr__(self):
        return self.__class__.__name__ + ':%s:%s:%s'%(self.dest, self.delete,
                self.specific)
    # }}}

class DeviceMenu(QMenu): # {{{

    fetch_annotations = pyqtSignal()
    disconnect_mounted_device = pyqtSignal()

    def __init__(self, parent=None):
        QMenu.__init__(self, parent)
        self.group = QActionGroup(self)
        self.actions = []
        self._memory = []

        self.set_default_menu = QMenu(_('Set default send to device action'))
        self.set_default_menu.setIcon(QIcon(I('config.png')))


        basic_actions = [
                ('main:', False, False,  I('reader.png'),
                    _('Send to main memory')),
                ('carda:0', False, False, I('sd.png'),
                    _('Send to storage card A')),
                ('cardb:0', False, False, I('sd.png'),
                    _('Send to storage card B')),
        ]

        delete_actions = [
                ('main:', True, False,   I('reader.png'),
                    _('Main Memory')),
                ('carda:0', True, False,  I('sd.png'),
                    _('Storage Card A')),
                ('cardb:0', True, False,  I('sd.png'),
                    _('Storage Card B')),
        ]

        specific_actions = [
                ('main:', False, True,  I('reader.png'),
                    _('Main Memory')),
                ('carda:0', False, True, I('sd.png'),
                    _('Storage Card A')),
                ('cardb:0', False, True, I('sd.png'),
                    _('Storage Card B')),
        ]

        later_menus = []

        for menu in (self, self.set_default_menu):
            for actions, desc in (
                    (basic_actions, ''),
                    (specific_actions, _('Send specific format to')),
                    (delete_actions, _('Send and delete from library')),
                    ):
                mdest = menu
                if actions is not basic_actions:
                    mdest = QMenu(desc)
                    self._memory.append(mdest)
                    later_menus.append(mdest)
                    if menu is self.set_default_menu:
                        menu.addMenu(mdest)
                        menu.addSeparator()

                for dest, delete, specific, icon, text in actions:
                    action = DeviceAction(dest, delete, specific, icon, text, self)
                    self._memory.append(action)
                    if menu is self.set_default_menu:
                        action.setCheckable(True)
                        action.setText(action.text())
                        self.group.addAction(action)
                    else:
                        action.a_s.connect(self.action_triggered)
                        self.actions.append(action)
                    mdest.addAction(action)
                if actions is basic_actions:
                    menu.addSeparator()

        da = config['default_send_to_device_action']
        done = False
        for action in self.group.actions():
            if repr(action) == da:
                action.setChecked(True)
                done = True
                break
        if not done:
            action = list(self.group.actions())[0]
            action.setChecked(True)
            config['default_send_to_device_action'] = repr(action)

        self.group.triggered.connect(self.change_default_action)
        self.addSeparator()

        self.addMenu(later_menus[0])
        self.addSeparator()

        mitem = self.addAction(QIcon(I('eject.png')), _('Eject device'))
        mitem.setEnabled(False)
        mitem.triggered.connect(lambda x : self.disconnect_mounted_device.emit())
        self.disconnect_mounted_device_action = mitem
        self.addSeparator()

        self.addMenu(self.set_default_menu)
        self.addSeparator()

        self.addMenu(later_menus[1])
        self.addSeparator()

        annot = self.addAction(_('Fetch annotations (experimental)'))
        annot.setEnabled(False)
        annot.triggered.connect(lambda x :
                self.fetch_annotations.emit())
        self.annotation_action = annot
        self.enable_device_actions(False)

    def change_default_action(self, action):
        config['default_send_to_device_action'] = repr(action)
        action.setChecked(True)

    def action_triggered(self, action):
        self.emit(SIGNAL('sync(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
                action.dest, action.delete, action.specific)

    def trigger_default(self, *args):
        r = config['default_send_to_device_action']
        for action in self.actions:
            if repr(action) == r:
                self.action_triggered(action)
                break

    def enable_device_actions(self, enable, card_prefix=(None, None),
            device=None):
        for action in self.actions:
            if action.dest in ('main:', 'carda:0', 'cardb:0'):
                if not enable:
                    action.setEnabled(False)
                else:
                    if action.dest == 'main:':
                        action.setEnabled(True)
                    elif action.dest == 'carda:0':
                        if card_prefix and card_prefix[0] != None:
                            action.setEnabled(True)
                        else:
                            action.setEnabled(False)
                    elif action.dest == 'cardb:0':
                        if card_prefix and card_prefix[1] != None:
                            action.setEnabled(True)
                        else:
                            action.setEnabled(False)

        annot_enable = enable and getattr(device, 'SUPPORTS_ANNOTATIONS', False)
        self.annotation_action.setEnabled(annot_enable)

    # }}}

class DeviceSignals(QObject): # {{{
    #: This signal is emitted once, after metadata is downloaded from the
    #: connected device.
    #: The sequence: gui.device_manager.is_device_connected will become True,
    #: and the device_connection_changed signal will be emitted,
    #: then sometime later gui.device_metadata_available will be signaled.
    #: This does not mean that there are no more jobs running. Automatic metadata
    #: management might have kicked off a sync_booklists to write new metadata onto
    #: the device, and that job might still be running when the signal is emitted.
    device_metadata_available = pyqtSignal()

    #: This signal is emitted once when the device is detected and once when
    #: it is disconnected. If the parameter is True, then it is a connection,
    #: otherwise a disconnection.
    device_connection_changed = pyqtSignal(object)

device_signals = DeviceSignals()
# }}}

class DeviceMixin(object): # {{{

    def __init__(self):
        self.device_error_dialog = error_dialog(self, _('Error'),
                _('Error communicating with device'), ' ')
        self.device_error_dialog.setModal(Qt.NonModal)
        self.device_manager = DeviceManager(FunctionDispatcher(self.device_detected),
                self.job_manager, Dispatcher(self.status_bar.show_message),
                Dispatcher(self.show_open_feedback),
                FunctionDispatcher(self.allow_connect))
        self.device_manager.start()
        self.device_manager.devices_initialized.wait()
        if tweaks['auto_connect_to_folder']:
            self.connect_to_folder_named(tweaks['auto_connect_to_folder'])

    def allow_connect(self, name, icon):
        return question_dialog(self, _('Manage the %s?')%name,
                _('Detected the <b>%s</b>. Do you want calibre to manage it?')%
                name, show_copy_button=False,
                override_icon=QIcon(icon))

    def debug_detection(self, done):
        self.debug_detection_callback = weakref.ref(done)
        self.device_manager.debug_detection(FunctionDispatcher(self.debug_detection_done))

    def debug_detection_done(self, job):
        d = self.debug_detection_callback()
        if d is not None:
            d(job)

    def show_open_feedback(self, devname, e):
        try:
            self.__of_dev_mem__ = d = e.custom_dialog(self)
        except NotImplementedError:
            self.__of_dev_mem__ = d = info_dialog(self, devname, e.feedback_msg)
        d.show()

    def auto_convert_question(self, msg, autos):
        autos = u'\n'.join(map(unicode, map(force_unicode, autos)))
        return self.ask_a_yes_no_question(
                _('No suitable formats'), msg,
                ans_when_user_unavailable=True,
                det_msg=autos, skip_dialog_name='auto_convert_before_send'
        )

    def set_default_thumbnail(self, height):
        img = I('book.png', data=True)
        self.default_thumbnail = thumbnail(img, height, height)

    def connect_to_folder_named(self, folder):
        if os.path.exists(folder) and os.path.isdir(folder):
            self.device_manager.mount_device(kls=FOLDER_DEVICE, kind='folder',
                    path=folder)

    def connect_to_folder(self):
        dir = choose_dir(self, 'Select Device Folder',
                             _('Select folder to open as device'))
        if dir is not None:
            self.device_manager.mount_device(kls=FOLDER_DEVICE, kind='folder', path=dir)

    def connect_to_bambook(self):
        self.device_manager.mount_device(kls=BAMBOOKWifi, kind='bambook',
                                         path=BAMBOOK.settings().extra_customization)

    def connect_to_itunes(self):
        self.device_manager.mount_device(kls=ITUNES_ASYNC, kind='itunes', path=None)

    # disconnect from both folder and itunes devices
    def disconnect_mounted_device(self):
        self.device_manager.umount_device()

    def configure_connected_device(self):
        if not self.device_manager.is_device_connected: return
        if self.job_manager.has_device_jobs(queued_also=True):
            return error_dialog(self, _('Running jobs'),
                    _('Cannot configure the device while there are running'
                        ' device jobs.'), show=True)
        dev = self.device_manager.connected_device
        cw = dev.config_widget()
        d = QDialog(self)
        d.setWindowTitle(_('Configure %s')%dev.get_gui_name())
        d.setWindowIcon(QIcon(I('config.png')))
        l = QVBoxLayout(d)
        d.setLayout(l)
        bb = QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel)
        bb.accepted.connect(d.accept)
        bb.rejected.connect(d.reject)
        l.addWidget(cw)
        l.addWidget(bb)
        def validate():
            if cw.validate():
                QDialog.accept(d)
        d.accept = validate
        if d.exec_() == d.Accepted:
            dev.save_settings(cw)
            do_restart = show_restart_warning(_('Restart calibre for the changes to %s'
                ' to be applied.')%dev.get_gui_name(), parent=self)
            if do_restart:
                self.quit(restart=True)

    def _sync_action_triggered(self, *args):
        m = getattr(self, '_sync_menu', None)
        if m is not None:
            m.trigger_default()

    def create_device_menu(self):
        self._sync_menu = DeviceMenu(self)
        self.iactions['Send To Device'].qaction.setMenu(self._sync_menu)
        self.iactions['Connect Share'].build_email_entries()
        self.connect(self._sync_menu,
                SIGNAL('sync(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
                self.dispatch_sync_event)
        self._sync_menu.fetch_annotations.connect(
                self.iactions['Fetch Annotations'].fetch_annotations)
        self._sync_menu.disconnect_mounted_device.connect(self.disconnect_mounted_device)
        self.iactions['Connect Share'].set_state(self.device_connected,
                None)
        if self.device_connected:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(True)
        else:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(False)

    def device_job_exception(self, job):
        '''
        Handle exceptions in threaded device jobs.
        '''
        if isinstance(getattr(job, 'exception', None), UserFeedback):
            ex = job.exception
            func = {UserFeedback.ERROR:error_dialog,
                    UserFeedback.WARNING:warning_dialog,
                    UserFeedback.INFO:info_dialog}[ex.level]
            return func(self, _('Failed'), ex.msg, det_msg=ex.details if
                    ex.details else '', show=True)

        try:
            if 'Could not read 32 bytes on the control bus.' in \
                    unicode(job.details):
                error_dialog(self, _('Error talking to device'),
                             _('There was a temporary error talking to the '
                             'device. Please unplug and reconnect the device '
                             'or reboot.')).show()
                return
        except:
            pass
        if getattr(job, 'exception', None).__class__.__name__ == 'MTPInvalidSendPathError':
            try:
                from calibre.gui2.device_drivers.mtp_config import SendError
                return SendError(self, job.exception).exec_()
            except:
                traceback.print_exc()
        try:
            prints(job.details, file=sys.stderr)
        except:
            pass
        if not self.device_error_dialog.isVisible():
            self.device_error_dialog.set_details(job.details)
            self.device_error_dialog.show()

    # Device connected {{{

    def set_device_menu_items_state(self, connected):
        self.iactions['Connect Share'].set_state(connected,
                self.device_manager.device)
        if connected:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(True)
            self._sync_menu.enable_device_actions(True,
                    self.device_manager.device.card_prefix(),
                    self.device_manager.device)
            self.eject_action.setEnabled(True)
        else:
            self._sync_menu.disconnect_mounted_device_action.setEnabled(False)
            self._sync_menu.enable_device_actions(False)
            self.eject_action.setEnabled(False)

    def device_detected(self, connected, device_kind):
        '''
        Called when a device is connected to the computer.
        '''
        # This can happen as this function is called in a queued connection and
        # the user could have yanked the device in the meantime
        if connected and not self.device_manager.is_device_connected:
            connected = False
        self.set_device_menu_items_state(connected)
        if connected:
            self.device_connected = device_kind
            self.device_manager.get_device_information(\
                    FunctionDispatcher(self.info_read))
            self.set_default_thumbnail(\
                    self.device_manager.device.THUMBNAIL_HEIGHT)
            self.status_bar.show_message(_('Device: ')+\
                self.device_manager.device.get_gui_name()+\
                        _(' detected.'), 3000)
            self.library_view.set_device_connected(self.device_connected)
            self.refresh_ondevice(reset_only=True)
        else:
            self.device_connected = None
            self.status_bar.device_disconnected()
            if self.current_view() != self.library_view:
                self.book_details.reset_info()
            self.location_manager.update_devices()
            self.bars_manager.update_bars()
            self.library_view.set_device_connected(self.device_connected)
            self.refresh_ondevice()
        device_signals.device_connection_changed.emit(connected)

    def info_read(self, job):
        '''
        Called once device information has been read.
        '''
        if job.failed:
            return self.device_job_exception(job)
        info, cp, fs = job.result
        self.location_manager.update_devices(cp, fs,
                self.device_manager.device.icon)
        self.bars_manager.update_bars()
        self.status_bar.device_connected(info[0])
        self.device_manager.books(FunctionDispatcher(self.metadata_downloaded),
                                  add_as_step_to_job=job)

    def metadata_downloaded(self, job):
        '''
        Called once metadata has been read for all books on the device.
        '''
        if job.failed:
            self.device_job_exception(job)
            return
        self.device_manager.slow_driveinfo()
        # set_books_in_library might schedule a sync_booklists job
        self.set_books_in_library(job.result, reset=True, add_as_step_to_job=job)
        mainlist, cardalist, cardblist = job.result
        self.memory_view.set_database(mainlist)
        self.memory_view.set_editable(self.device_manager.device.CAN_SET_METADATA,
                                      self.device_manager.device.BACKLOADING_ERROR_MESSAGE
                                      is None)
        self.card_a_view.set_database(cardalist)
        self.card_a_view.set_editable(self.device_manager.device.CAN_SET_METADATA,
                                      self.device_manager.device.BACKLOADING_ERROR_MESSAGE
                                      is None)
        self.card_b_view.set_database(cardblist)
        self.card_b_view.set_editable(self.device_manager.device.CAN_SET_METADATA,
                                      self.device_manager.device.BACKLOADING_ERROR_MESSAGE
                                      is None)
        self.sync_news()
        self.sync_catalogs()
        self.refresh_ondevice()
        device_signals.device_metadata_available.emit()

    def refresh_ondevice(self, reset_only=False):
        '''
        Force the library view to refresh, taking into consideration new
        device books information
        '''
        with self.library_view.preserve_state():
            self.book_on_device(None, reset=True)
            if reset_only:
                return
            self.library_view.model().refresh_ondevice()

    # }}}

    def remove_paths(self, paths):
        return self.device_manager.delete_books(
                FunctionDispatcher(self.books_deleted), paths)

    def books_deleted(self, job):
        '''
        Called once deletion is done on the device
        '''
        for view in (self.memory_view, self.card_a_view, self.card_b_view):
            view.model().deletion_done(job, job.failed)
        if job.failed:
            self.device_job_exception(job)
            return

        dm = self.iactions['Remove Books'].delete_memory
        if dm.has_key(job):
            paths, model = dm.pop(job)
            self.device_manager.remove_books_from_metadata(paths,
                    self.booklists())
            model.paths_deleted(paths)
        # Force recomputation the library's ondevice info. We need to call
        # set_books_in_library even though books were not added because
        # the deleted book might have been an exact match. Upload the booklists
        # if set_books_in_library did not.
        if not self.set_books_in_library(self.booklists(), reset=True, add_as_step_to_job=job):
            self.upload_booklists(job)
        # We need to reset the ondevice flags in the library. Use a big hammer,
        # so we don't need to worry about whether some succeeded or not.
        self.refresh_ondevice(reset_only=False)

    def dispatch_sync_event(self, dest, delete, specific):
        rows = self.library_view.selectionModel().selectedRows()
        if not rows or len(rows) == 0:
            error_dialog(self, _('No books'), _('No books')+' '+\
                    _('selected to send')).exec_()
            return

        fmt = None
        if specific:
            if (not self.device_connected or not self.device_manager or
                    self.device_manager.device is None):
                error_dialog(self, _('No device'),
                        _('No device connected'), show=True)
                return
            formats = []
            aval_out_formats = available_output_formats()
            format_count = {}
            for row in rows:
                fmts = self.library_view.model().db.formats(row.row())
                if fmts:
                    for f in fmts.split(','):
                        f = f.lower()
                        if format_count.has_key(f):
                            format_count[f] += 1
                        else:
                            format_count[f] = 1
            for f in self.device_manager.device.settings().format_map:
                if f in format_count.keys():
                    formats.append((f, _('%(num)i of %(total)i Books') % dict(
                        num=format_count[f], total=len(rows)),
                        True if f in aval_out_formats else False))
                elif f in aval_out_formats:
                    formats.append((f, _('0 of %i Books') % len(rows), True))
            d = ChooseFormatDeviceDialog(self, _('Choose format to send to device'), formats)
            if d.exec_() != QDialog.Accepted:
                return
            if d.format():
                fmt = d.format().lower()
        dest, sub_dest = dest.split(':')
        if dest in ('main', 'carda', 'cardb'):
            if not self.device_connected or not self.device_manager:
                error_dialog(self, _('No device'),
                        _('Cannot send: No device is connected')).exec_()
                return
            if dest == 'carda' and not self.device_manager.has_card():
                error_dialog(self, _('No card'),
                        _('Cannot send: Device has no storage card')).exec_()
                return
            if dest == 'cardb' and not self.device_manager.has_card():
                error_dialog(self, _('No card'),
                        _('Cannot send: Device has no storage card')).exec_()
                return
            if dest == 'main':
                on_card = None
            else:
                on_card = dest
            self.sync_to_device(on_card, delete, fmt)
        elif dest == 'mail':
            sub_dest_parts = sub_dest.split(';')
            while len(sub_dest_parts) < 3:
                sub_dest_parts.append('')
            to = sub_dest_parts[0]
            fmts = sub_dest_parts[1]
            subject = ';'.join(sub_dest_parts[2:])
            fmts = [x.strip().lower() for x in fmts.split(',')]
            self.send_by_mail(to, fmts, delete, subject=subject)

    def cover_to_thumbnail(self, data):
        if self.device_manager.device and \
                hasattr(self.device_manager.device, 'THUMBNAIL_WIDTH'):
            try:
                return thumbnail(data,
                                 self.device_manager.device.THUMBNAIL_WIDTH,
                                 self.device_manager.device.THUMBNAIL_HEIGHT,
                                 preserve_aspect_ratio=False)
            except:
                pass
            return
        ht = self.device_manager.device.THUMBNAIL_HEIGHT \
                if self.device_manager else DevicePlugin.THUMBNAIL_HEIGHT
        try:
            return thumbnail(data, ht, ht)
        except:
            pass

    def sync_catalogs(self, send_ids=None, do_auto_convert=True):
        if self.device_connected:
            settings = self.device_manager.device.settings()
            ids = list(dynamic.get('catalogs_to_be_synced', set([]))) if send_ids is None else send_ids
            ids = [id for id in ids if self.library_view.model().db.has_id(id)]
            files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(
                                ids, settings.format_map,
                                exclude_auto=do_auto_convert)
            auto = []
            if do_auto_convert and _auto_ids:
                for id in _auto_ids:
                    dbfmts = self.library_view.model().db.formats(id, index_is_id=True)
                    formats = [] if dbfmts is None else \
                        [f.lower() for f in dbfmts.split(',')]
                    if set(formats).intersection(available_input_formats()) \
                            and set(settings.format_map).intersection(available_output_formats()):
                        auto.append(id)
            if auto:
                format = None
                for fmt in settings.format_map:
                    if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        format = fmt
                        break
                if format is not None:
                    autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto]
                    if self.auto_convert_question(
                        _('Auto convert the following books before uploading to '
                            'the device?'), autos):
                        self.iactions['Convert Books'].auto_convert_catalogs(auto, format)
            files = [f for f in files if f is not None]
            if not files:
                dynamic.set('catalogs_to_be_synced', set([]))
                return
            metadata = self.library_view.model().metadata_for(ids)
            names = []
            for mi in metadata:
                prefix = ascii_filename(mi.title)
                if not isinstance(prefix, unicode):
                    prefix = prefix.decode(preferred_encoding, 'replace')
                prefix = ascii_filename(prefix)
                names.append('%s_%d%s'%(prefix, id,
                    os.path.splitext(f)[1]))
                if mi.cover and os.access(mi.cover, os.R_OK):
                    mi.thumbnail = self.cover_to_thumbnail(open(mi.cover,
                        'rb').read())
            dynamic.set('catalogs_to_be_synced', set([]))
            if files:
                remove = []
                space = { self.location_manager.free[0] : None,
                    self.location_manager.free[1] : 'carda',
                    self.location_manager.free[2] : 'cardb' }
                on_card = space.get(sorted(space.keys(), reverse=True)[0], None)
                self.upload_books(files, names, metadata,
                        on_card=on_card,
                        memory=[files, remove])
                self.status_bar.show_message(_('Sending catalogs to device.'), 5000)


    @dynamic_property
    def news_to_be_synced(self):
        doc = 'Set of ids to be sent to device'
        def fget(self):
            ans = []
            try:
                ans = self.library_view.model().db.prefs.get('news_to_be_synced',
                        [])
            except:
                import traceback
                traceback.print_exc()
            return set(ans)

        def fset(self, ids):
            try:
                self.library_view.model().db.prefs.set('news_to_be_synced',
                        list(ids))
            except:
                import traceback
                traceback.print_exc()

        return property(fget=fget, fset=fset, doc=doc)


    def sync_news(self, send_ids=None, do_auto_convert=True):
        if self.device_connected:
            del_on_upload = config['delete_news_from_library_on_upload']
            settings = self.device_manager.device.settings()
            ids = list(self.news_to_be_synced) if send_ids is None else send_ids
            ids = [id for id in ids if self.library_view.model().db.has_id(id)]
            files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(
                                ids, settings.format_map,
                                exclude_auto=do_auto_convert)
            auto = []
            if do_auto_convert and _auto_ids:
                for id in _auto_ids:
                    dbfmts = self.library_view.model().db.formats(id, index_is_id=True)
                    formats = [] if dbfmts is None else \
                        [f.lower() for f in dbfmts.split(',')]
                    if set(formats).intersection(available_input_formats()) \
                            and set(settings.format_map).intersection(available_output_formats()):
                        auto.append(id)
            if auto:
                format = None
                for fmt in settings.format_map:
                    if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        format = fmt
                        break
                if format is not None:
                    autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto]
                    if self.auto_convert_question(
                        _('Auto convert the following books before uploading to '
                            'the device?'), autos):
                        self.iactions['Convert Books'].auto_convert_news(auto, format)
            files = [f for f in files if f is not None]
            if not files:
                self.news_to_be_synced = set([])
                return
            metadata = self.library_view.model().metadata_for(ids)
            names = []
            for mi in metadata:
                prefix = ascii_filename(mi.title)
                if not isinstance(prefix, unicode):
                    prefix = prefix.decode(preferred_encoding, 'replace')
                prefix = ascii_filename(prefix)
                names.append('%s_%d%s'%(prefix, id,
                    os.path.splitext(f)[1]))
                if mi.cover and os.access(mi.cover, os.R_OK):
                    mi.thumbnail = self.cover_to_thumbnail(open(mi.cover,
                        'rb').read())
            self.news_to_be_synced = set([])
            if config['upload_news_to_device'] and files:
                remove = ids if del_on_upload else []
                space = { self.location_manager.free[0] : None,
                    self.location_manager.free[1] : 'carda',
                    self.location_manager.free[2] : 'cardb' }
                on_card = space.get(sorted(space.keys(), reverse=True)[0], None)
                try:
                    total_size = sum([os.stat(f).st_size for f in files])
                except:
                    try:
                        import traceback
                        traceback.print_exc()
                    except:
                        pass
                    total_size = self.location_manager.free[0]
                loc = tweaks['send_news_to_device_location']
                loc_index = {"carda": 1, "cardb": 2}.get(loc, 0)
                if self.location_manager.free[loc_index] > total_size + (1024**2):
                    # Send news to main memory if enough space available
                    # as some devices like the Nook Color cannot handle
                    # periodicals on SD cards properly
                    on_card = loc if loc in ('carda', 'cardb') else None
                self.upload_books(files, names, metadata,
                        on_card=on_card,
                        memory=[files, remove])
                self.status_bar.show_message(_('Sending news to device.'), 5000)


    def sync_to_device(self, on_card, delete_from_library,
            specific_format=None, send_ids=None, do_auto_convert=True):
        ids = [self.library_view.model().id(r) \
               for r in self.library_view.selectionModel().selectedRows()] \
                                if send_ids is None else send_ids
        if not self.device_manager or not ids or len(ids) == 0 or \
                not self.device_manager.is_device_connected:
            return

        settings = self.device_manager.device.settings()

        _files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(ids,
                                    settings.format_map,
                                    specific_format=specific_format,
                                    exclude_auto=do_auto_convert)
        if do_auto_convert:
            ok_ids = list(set(ids).difference(_auto_ids))
            ids = [i for i in ids if i in ok_ids]
        else:
            _auto_ids = []

        metadata = self.library_view.model().metadata_for(ids)
        ids = iter(ids)
        for mi in metadata:
            if mi.cover and os.access(mi.cover, os.R_OK):
                mi.thumbnail = self.cover_to_thumbnail(open(mi.cover, 'rb').read())
        imetadata = iter(metadata)

        bad, good, gf, names, remove_ids = [], [], [], [], []
        for f in _files:
            mi = imetadata.next()
            id = ids.next()
            if f is None:
                bad.append(mi.title)
            else:
                remove_ids.append(id)
                good.append(mi)
                gf.append(f)
                t = mi.title
                if not t:
                    t = _('Unknown')
                a = mi.format_authors()
                if not a:
                    a = _('Unknown')
                prefix = ascii_filename(t+' - '+a)
                if not isinstance(prefix, unicode):
                    prefix = prefix.decode(preferred_encoding, 'replace')
                prefix = ascii_filename(prefix)
                names.append('%s_%d%s'%(prefix, id, os.path.splitext(f)[1]))
        remove = remove_ids if delete_from_library else []
        self.upload_books(gf, names, good, on_card, memory=(_files, remove))
        self.status_bar.show_message(_('Sending books to device.'), 5000)

        auto = []
        if _auto_ids != []:
            for id in _auto_ids:
                if specific_format == None:
                    formats = self.library_view.model().db.formats(id, index_is_id=True)
                    formats = formats.split(',') if formats is not None else []
                    formats = [f.lower().strip() for f in formats]
                    if list(set(formats).intersection(available_input_formats())) != [] and list(set(settings.format_map).intersection(available_output_formats())) != []:
                        auto.append(id)
                    else:
                        bad.append(self.library_view.model().db.title(id, index_is_id=True))
                else:
                    if specific_format in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        auto.append(id)
                    else:
                        bad.append(self.library_view.model().db.title(id, index_is_id=True))

        if auto != []:
            format = specific_format if specific_format in \
                            list(set(settings.format_map).intersection(set(available_output_formats()))) \
                            else None
            if not format:
                for fmt in settings.format_map:
                    if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))):
                        format = fmt
                        break
            if not format:
                bad += auto
            else:
                autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto]
                if self.auto_convert_question(
                    _('Auto convert the following books before uploading to '
                        'the device?'), autos):
                    self.iactions['Convert Books'].auto_convert(auto, on_card, format)

        if bad:
            bad = '\n'.join('%s'%(i,) for i in bad)
            d = warning_dialog(self, _('No suitable formats'),
                    _('Could not upload the following books to the device, '
                'as no suitable formats were found. Convert the book(s) to a '
                'format supported by your device first.'
                ), bad)
            d.exec_()

    def upload_dirtied_booklists(self):
        '''
        Upload metadata to device.
        '''
        plugboards = self.library_view.model().db.prefs.get('plugboards', {})
        self.device_manager.sync_booklists(Dispatcher(lambda x: x),
                                           self.booklists(), plugboards)

    def upload_booklists(self, add_as_step_to_job=None):
        '''
        Upload metadata to device.
        '''
        plugboards = self.library_view.model().db.prefs.get('plugboards', {})
        self.device_manager.sync_booklists(FunctionDispatcher(self.metadata_synced),
                                           self.booklists(), plugboards,
                                           add_as_step_to_job=add_as_step_to_job)

    def metadata_synced(self, job):
        '''
        Called once metadata has been uploaded.
        '''
        if job.failed:
            self.device_job_exception(job)
            return
        cp, fs = job.result
        self.location_manager.update_devices(cp, fs,
                self.device_manager.device.icon)
        # reset the views so that up-to-date info is shown. These need to be
        # here because some drivers update collections in sync_booklists
        self.memory_view.reset()
        self.card_a_view.reset()
        self.card_b_view.reset()

    def _upload_collections(self, job):
        if job.failed:
            self.device_job_exception(job)

    def upload_collections(self, booklist, view=None, oncard=None):
        return self.device_manager.upload_collections(self._upload_collections,
                                                       booklist, oncard)

    def upload_books(self, files, names, metadata, on_card=None, memory=None):
        '''
        Upload books to device.
        :param files: List of either paths to files or file like objects
        '''
        titles = [i.title for i in metadata]
        plugboards = self.library_view.model().db.prefs.get('plugboards', {})
        job = self.device_manager.upload_books(
                FunctionDispatcher(self.books_uploaded),
                files, names, on_card=on_card,
                metadata=metadata, titles=titles, plugboards=plugboards
              )
        self.upload_memory[job] = (metadata, on_card, memory, files)

    def books_uploaded(self, job):
        '''
        Called once books have been uploaded.
        '''
        metadata, on_card, memory, files = self.upload_memory.pop(job)

        if job.exception is not None:
            if isinstance(job.exception, FreeSpaceError):
                where = 'in main memory.' if 'memory' in str(job.exception) \
                        else 'on the storage card.'
                titles = '\n'.join(['<li>'+mi.title+'</li>' \
                                    for mi in metadata])
                d = error_dialog(self, _('No space on device'),
                                 _('<p>Cannot upload books to device there '
                                 'is no more free space available ')+where+
                                 '</p>\n<ul>%s</ul>'%(titles,))
                d.exec_()
            elif isinstance(job.exception, WrongDestinationError):
                error_dialog(self, _('Incorrect destination'),
                        unicode(job.exception), show=True)
            else:
                self.device_job_exception(job)
            return

        try:
            self.device_manager.add_books_to_metadata(job.result,
                    metadata, self.booklists())
        except:
            traceback.print_exc()
            raise

        books_to_be_deleted = []
        if memory and memory[1]:
            books_to_be_deleted = memory[1]
            self.library_view.model().delete_books_by_id(books_to_be_deleted)

        # There are some cases where sending a book to the device overwrites a
        # book already there with a different book. This happens frequently in
        # news. When this happens, the book match indication will be wrong
        # because the UUID changed. Force both the device and the library view
        # to refresh the flags. Set_books_in_library could upload the booklists.
        # If it does not, then do it here.
        if not self.set_books_in_library(self.booklists(), reset=True, add_as_step_to_job=job):
            self.upload_booklists(job)
        self.refresh_ondevice()

        view = self.card_a_view if on_card == 'carda' else \
            self.card_b_view if on_card == 'cardb' else self.memory_view
        view.model().resort(reset=False)
        view.model().research()
        if files:
            for f in files:
                # Remove temporary files
                try:
                    rem = not getattr(
                            self.device_manager.device,
                            'KEEP_TEMP_FILES_AFTER_UPLOAD', False)
                    if rem and 'caltmpfmt.' in f:
                        os.remove(f)
                except:
                    pass

    def book_on_device(self, id, reset=False):
        '''
        Return an indication of whether the given book represented by its db id
        is on the currently connected device. It returns a 5 element list. The
        first three elements represent memory locations main, carda, and cardb,
        and are true if the book is identifiably in that memory. The fourth
        is a count of how many instances of the book were found across all
        the memory locations. The fifth is a set of paths to the
        matching books on the device.
        '''
        loc = [None, None, None, 0, set([])]

        if reset:
            self.book_db_id_cache = None
            self.book_db_id_counts = None
            self.book_db_uuid_path_map = None
            return

        if not self.device_manager.is_device_connected or \
                        not hasattr(self, 'db_book_uuid_cache'):
            return loc

        if self.book_db_id_cache is None:
            self.book_db_id_cache = []
            self.book_db_id_counts = {}
            self.book_db_uuid_path_map = {}
            for i, l in enumerate(self.booklists()):
                self.book_db_id_cache.append(set())
                for book in l:
                    db_id = getattr(book, 'application_id', None)
                    if db_id is not None:
                        # increment the count of books on the device with this
                        # db_id.
                        self.book_db_id_cache[i].add(db_id)
                        if db_id not in self.book_db_uuid_path_map:
                            self.book_db_uuid_path_map[db_id] = set()
                        if getattr(book, 'lpath', False):
                            self.book_db_uuid_path_map[db_id].add(book.lpath)
                        c = self.book_db_id_counts.get(db_id, 0)
                        self.book_db_id_counts[db_id] = c + 1

        for i, l in enumerate(self.booklists()):
            if id in self.book_db_id_cache[i]:
                loc[i] = True
                loc[3] = self.book_db_id_counts.get(id, 0)
                loc[4] |= self.book_db_uuid_path_map[id]
        return loc

    def set_books_in_library(self, booklists, reset=False, add_as_step_to_job=None):
        '''
        Set the ondevice indications in the device database.
        This method should be called before book_on_device is called, because
        it sets the application_id for matched books. Book_on_device uses that
        to both speed up matching and to count matches.
        '''

        if not self.device_manager.is_device_connected:
            return False

        # It might be possible to get here without having initialized the
        # library view. In this case, simply give up
        try:
            db = self.library_view.model().db
        except:
            return False

        string_pat = re.compile('(?u)\W|[_]')
        def clean_string(x):
            x = x.lower() if x else ''
            return string_pat.sub('', x)

        update_metadata = device_prefs['manage_device_metadata'] == 'on_connect'

        get_covers = False
        desired_thumbnail_height = 0
        if update_metadata and self.device_manager.is_device_connected:
            if self.device_manager.device.WANTS_UPDATED_THUMBNAILS:
                get_covers = True
                desired_thumbnail_height = self.device_manager.device.THUMBNAIL_HEIGHT

        # Force a reset if the caches are not initialized
        if reset or not hasattr(self, 'db_book_title_cache'):
            # Build a cache (map) of the library, so the search isn't On**2
            db_book_title_cache = {}
            db_book_uuid_cache = {}

            for id_ in db.data.iterallids():
                title = clean_string(db.title(id_, index_is_id=True))
                if title not in db_book_title_cache:
                    db_book_title_cache[title] = \
                                {'authors':{}, 'author_sort':{}, 'db_ids':{}}
                # If there are multiple books in the library with the same title
                # and author, then remember the last one. That is OK, because as
                # we can't tell the difference between the books, one is as good
                # as another.
                authors = clean_string(db.authors(id_, index_is_id=True))
                if authors:
                    db_book_title_cache[title]['authors'][authors] = id_
                if db.author_sort(id_, index_is_id=True):
                    aus = clean_string(db.author_sort(id_, index_is_id=True))
                    db_book_title_cache[title]['author_sort'][aus] = id_
                db_book_title_cache[title]['db_ids'][id_] = id_
                db_book_uuid_cache[db.uuid(id_, index_is_id=True)] = id_
            self.db_book_title_cache = db_book_title_cache
            self.db_book_uuid_cache = db_book_uuid_cache

        # Now iterate through all the books on the device, setting the
        # in_library field. If the UUID matches a book in the library, then
        # do not consider that book for other matching. In all cases set
        # the application_id to the db_id of the matching book. This value
        # will be used by books_on_device to indicate matches. While we are
        # going by, update the metadata for a book if automatic management is on

        def update_book(id_, book) :
            if not update_metadata:
                return
            mi = db.get_metadata(id_, index_is_id=True, get_cover=get_covers)
            book.smart_update(mi, replace_metadata=True)
            if get_covers and desired_thumbnail_height != 0:
                if book.cover and os.access(book.cover, os.R_OK):
                    book.thumbnail = self.cover_to_thumbnail(open(book.cover, 'rb').read())
                else:
                    book.thumbnail = self.default_thumbnail

        def updateq(id_, book):
            try:
                return (update_metadata and
                        (db.metadata_last_modified(id_, index_is_id=True) !=
                         getattr(book, 'last_modified', None) or
                         (isinstance(getattr(book, 'thumbnail', None), (list, tuple))
                          and max(book.thumbnail[0], book.thumbnail[1]) != desired_thumbnail_height
                         )
                        )
                       )
            except:
                return True

        for booklist in booklists:
            for book in booklist:
                book.in_library = None
                if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
                    id_ = db_book_uuid_cache[book.uuid]
                    if updateq(id_, book):
                        update_book(id_, book)
                    book.in_library = 'UUID'
                    # ensure that the correct application_id is set
                    book.application_id = id_
                    continue
                # No UUID exact match. Try metadata matching.
                book_title = clean_string(book.title)
                d = self.db_book_title_cache.get(book_title, None)
                if d is not None:
                    # At this point we know that the title matches. The book
                    # will match if any of the db_id, author, or author_sort
                    # also match.
                    if getattr(book, 'application_id', None) in d['db_ids']:
                        id_ = getattr(book, 'application_id', None)
                        update_book(id_, book)
                        book.in_library = 'APP_ID'
                        # app_id already matches a db_id. No need to set it.
                        continue
                    # Sonys know their db_id independent of the application_id
                    # in the metadata cache. Check that as well.
                    if getattr(book, 'db_id', None) in d['db_ids']:
                        update_book(book.db_id, book)
                        book.in_library = 'DB_ID'
                        book.application_id = book.db_id
                        continue
                    # We now know that the application_id is not right. Set it
                    # to None to prevent book_on_device from accidentally
                    # matching on it. It will be set to a correct value below if
                    # the book is matched with one in the library
                    book.application_id = None
                    if book.authors:
                        # Compare against both author and author sort, because
                        # either can appear as the author
                        book_authors = clean_string(authors_to_string(book.authors))
                        if book_authors in d['authors']:
                            id_ = d['authors'][book_authors]
                            update_book(id_, book)
                            book.in_library = 'AUTHOR'
                            book.application_id = id_
                        elif book_authors in d['author_sort']:
                            id_ = d['author_sort'][book_authors]
                            update_book(id_, book)
                            book.in_library = 'AUTH_SORT'
                            book.application_id = id_
                else:
                    # Book definitely not matched. Clear its application ID
                    book.application_id = None
                # Set author_sort if it isn't already
                asort = getattr(book, 'author_sort', None)
                if not asort and book.authors:
                    book.author_sort = self.library_view.model().db.\
                                author_sort_from_authors(book.authors)

        if update_metadata:
            if self.device_manager.is_device_connected:
                plugboards = self.library_view.model().db.prefs.get('plugboards', {})
                self.device_manager.sync_booklists(
                                    FunctionDispatcher(self.metadata_synced), booklists,
                                    plugboards, add_as_step_to_job)
        return update_metadata
    # }}}

