Initial import

1 files changed, 589 insertions, 0 deletions

diff --git a/src/z3950connection.cpp b/src/z3950connection.cpp
new file mode 100644
index 0000000..ada2a1d
--- /dev/null
+++ b/src/z3950connection.cpp
@@ -0,0 +1,589 @@
+/***************************************************************************
+    copyright            : (C) 2005-2006 by Robby Stephenson
+    email                : $EMAIL
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This file has been modified to match the requirements of KBibTeX.     *
+ *   In case of problems or bugs arising from this implementation, please  *
+ *   contact the KBibTeX team first.                                       *
+ *                             Thomas Fischer <[email protected]>  *
+ *                                                                         *
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of version 2 of the GNU General Public License as  *
+ *   published by the Free Software Foundation;                            *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "z3950connection.h"
+// #include "z3950fetcher.h"
+#include "messagehandler.h"
+#include "latin1literal.h"
+#include <kdebug.h>
+#include "iso5426converter.h"
+#include "iso6937converter.h"
+
+#include <qapplication.h>
+
+#include <config.h>
+
+#ifdef HAVE_YAZ
+extern "C"
+{
+#include <yaz/zoom.h>
+#include <yaz/marcdisp.h>
+#include <yaz/yaz-version.h>
+}
+#endif
+
+#include <klocale.h>
+
+#include <qfile.h>
+
+namespace
+{
+    static const size_t Z3950_DEFAULT_MAX_RECORDS = 20;
+}
+
+using KBibTeX::Z3950ResultFound;
+using KBibTeX::Z3950Connection;
+
+Z3950ResultFound::Z3950ResultFound( const QString& s ) : QCustomEvent( uid() )
+        , m_result( QDeepCopy<QString>( s ) )
+{
+    ++Z3950Connection::resultsLeft;
+}
+
+Z3950ResultFound::~Z3950ResultFound()
+{
+    --Z3950Connection::resultsLeft;
+}
+
+class Z3950Connection::Private
+{
+public:
+    Private() {}
+#ifdef HAVE_YAZ
+    ~Private()
+    {
+        ZOOM_options_destroy( conn_opt );
+        ZOOM_connection_destroy( conn );
+    };
+
+    ZOOM_options conn_opt;
+    ZOOM_connection conn;
+#endif
+};
+
+int Z3950Connection::resultsLeft = 0;
+
+// since the character set goes into a yaz api call
+// I'm paranoid about user insertions, so just grab 64
+// characters at most
+Z3950Connection::Z3950Connection( QObject* fetcher,
+                                  const QString& host,
+                                  uint port,
+                                  const QString& dbname,
+                                  const QString& sourceCharSet,
+                                  const QString& syntax,
+                                  const QString& esn )
+        : QThread()
+        , d( new Private() )
+        , m_connected( false )
+        , m_aborted( false )
+        , m_fetcher( fetcher )
+        , m_host( QDeepCopy<QString>( host ) )
+        , m_port( port )
+        , m_dbname( QDeepCopy<QString>( dbname ) )
+        , m_sourceCharSet( QDeepCopy<QString>( sourceCharSet.left( 64 ) ) )
+        , m_syntax( QDeepCopy<QString>( syntax ) )
+        , m_esn( QDeepCopy<QString>( esn ) )
+        , m_start( 0 )
+        , m_limit( Z3950_DEFAULT_MAX_RECORDS )
+        , m_hasMore( false )
+{
+}
+
+Z3950Connection::~Z3950Connection()
+{
+    m_connected = false;
+    delete d;
+    d = 0;
+}
+
+void Z3950Connection::reset()
+{
+    m_start = 0;
+    m_limit = Z3950_DEFAULT_MAX_RECORDS;
+}
+
+void Z3950Connection::setQuery( const QString& query_, unsigned int numHits )
+{
+    m_pqn = QDeepCopy<QString>( query_ );
+    m_limit = Z3950_DEFAULT_MAX_RECORDS < numHits ? Z3950_DEFAULT_MAX_RECORDS : numHits;
+}
+
+void Z3950Connection::setUserPassword( const QString& user_, const QString& pword_ )
+{
+    m_user = QDeepCopy<QString>( user_ );
+    m_password = QDeepCopy<QString>( pword_ );
+}
+
+void Z3950Connection::run()
+{
+//  kdDebug() << "Z3950Connection::run() - " << m_fetcher->source() << endl;
+    m_aborted = false;
+    m_hasMore = false;
+    resultsLeft = 0;
+#ifdef HAVE_YAZ
+
+    if ( !makeConnection() )
+    {
+        done();
+        return;
+    }
+
+    ZOOM_query query = ZOOM_query_create();
+    kdDebug() << "Z3950Connection::run() - pqn = " << toCString( m_pqn ) << endl;
+    int errcode = ZOOM_query_prefix( query, toCString( m_pqn ) );
+    if ( errcode != 0 )
+    {
+        kdDebug() << "Z3950Connection::run() - query error: " << m_pqn << endl;
+        ZOOM_query_destroy( query );
+        QString s = i18n( "Query error!" );
+        s += ' ' + m_pqn;
+        done( s, MessageHandler::Error );
+        return;
+    }
+
+    ZOOM_resultset resultSet = ZOOM_connection_search( d->conn, query );
+
+    // check abort status
+    if ( m_aborted )
+    {
+        done();
+        return;
+    }
+
+    // I know the LOC wants the syntax = "xml" and esn = "mods"
+    // to get MODS data, that seems a bit odd...
+    // esn only makes sense for marc and grs-1
+    // if syntax is mods, set esn to mods too
+    QCString type = "raw";
+    if ( m_syntax == Latin1Literal( "mods" ) )
+    {
+        m_syntax = QString::fromLatin1( "xml" );
+        ZOOM_resultset_option_set( resultSet, "elementSetName", "mods" );
+        type = "xml";
+    }
+    else
+    {
+        ZOOM_resultset_option_set( resultSet, "elementSetName", m_esn.latin1() );
+    }
+    ZOOM_resultset_option_set( resultSet, "start", QCString().setNum( m_start ) );
+    ZOOM_resultset_option_set( resultSet, "count", QCString().setNum( m_limit - m_start ) );
+    // search in default syntax, unless syntax is already set
+    if ( !m_syntax.isEmpty() )
+    {
+        ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", m_syntax.latin1() );
+    }
+
+    const char* errmsg;
+    const char* addinfo;
+    errcode = ZOOM_connection_error( d->conn, &errmsg, &addinfo );
+    if ( errcode != 0 )
+    {
+        ZOOM_resultset_destroy( resultSet );
+        ZOOM_query_destroy( query );
+        m_connected = false;
+
+        QString s = i18n( "Connection search error %1: %2" ).arg( errcode ).arg( toString( errmsg ) );
+        if ( !QCString( addinfo ).isEmpty() )
+        {
+            s += " (" + toString( addinfo ) + ")";
+        }
+        kdDebug() << "Z3950Connection::run() - " << s << endl;
+        done( s, MessageHandler::Error );
+        return;
+    }
+
+    const size_t numResults = ZOOM_resultset_size( resultSet );
+
+    QString newSyntax = m_syntax;
+    if ( numResults > 0 )
+    {
+        kdDebug() << "Z3950Connection::run() - current syntax is " << m_syntax << " (" << numResults << " results)" << endl;
+        // so now we know that results exist, might have to check syntax
+        int len;
+        ZOOM_record rec = ZOOM_resultset_record( resultSet, 0 );
+        // want raw unless it's mods
+        ZOOM_record_get( rec, type, &len );
+        if ( len > 0 && m_syntax.isEmpty() )
+        {
+            newSyntax = QString::fromLatin1( ZOOM_record_get( rec, "syntax", &len ) ).lower();
+            kdDebug() << "Z3950Connection::run() - syntax guess is " << newSyntax << endl;
+            if ( newSyntax == Latin1Literal( "mods" ) || newSyntax == Latin1Literal( "xml" ) )
+            {
+                m_syntax = QString::fromLatin1( "xml" );
+                ZOOM_resultset_option_set( resultSet, "elementSetName", "mods" );
+            }
+            else if ( newSyntax == Latin1Literal( "grs-1" ) )
+            {
+                // if it's defaulting to grs-1, go ahead and change it to try to get a marc
+                // record since grs-1 is a last resort for us
+                newSyntax.truncate( 0 );
+            }
+        }
+        // right now, we just understand mods, unimarc, marc21/usmarc, and grs-1
+        if ( newSyntax != Latin1Literal( "xml" ) &&
+                newSyntax != Latin1Literal( "usmarc" ) &&
+                newSyntax != Latin1Literal( "marc21" ) &&
+                newSyntax != Latin1Literal( "unimarc" ) &&
+                newSyntax != Latin1Literal( "grs-1" ) )
+        {
+            kdDebug() << "Z3950Connection::run() - changing z39.50 syntax to MODS" << endl;
+            newSyntax = QString::fromLatin1( "xml" );
+            ZOOM_resultset_option_set( resultSet, "elementSetName", "mods" );
+            ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", newSyntax.latin1() );
+            rec = ZOOM_resultset_record( resultSet, 0 );
+            ZOOM_record_get( rec, "xml", &len );
+            if ( len == 0 )
+            {
+                // change set name back
+                ZOOM_resultset_option_set( resultSet, "elementSetName", m_esn.latin1() );
+                newSyntax = QString::fromLatin1( "usmarc" ); // try usmarc
+                kdDebug() << "Z3950Connection::run() - changing z39.50 syntax to USMARC" << endl;
+                ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", newSyntax.latin1() );
+                rec = ZOOM_resultset_record( resultSet, 0 );
+                ZOOM_record_get( rec, "raw", &len );
+            }
+            if ( len == 0 )
+            {
+                newSyntax = QString::fromLatin1( "marc21" ); // try marc21
+                kdDebug() << "Z3950Connection::run() - changing z39.50 syntax to MARC21" << endl;
+                ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", newSyntax.latin1() );
+                rec = ZOOM_resultset_record( resultSet, 0 );
+                ZOOM_record_get( rec, "raw", &len );
+            }
+            if ( len == 0 )
+            {
+                newSyntax = QString::fromLatin1( "unimarc" ); // try unimarc
+                kdDebug() << "Z3950Connection::run() - changing z39.50 syntax to UNIMARC" << endl;
+                ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", newSyntax.latin1() );
+                rec = ZOOM_resultset_record( resultSet, 0 );
+                ZOOM_record_get( rec, "raw", &len );
+            }
+            if ( len == 0 )
+            {
+                newSyntax = QString::fromLatin1( "grs-1" ); // try grs-1
+                kdDebug() << "Z3950Connection::run() - changing z39.50 syntax to GRS-1" << endl;
+                ZOOM_resultset_option_set( resultSet, "preferredRecordSyntax", newSyntax.latin1() );
+                rec = ZOOM_resultset_record( resultSet, 0 );
+                ZOOM_record_get( rec, "raw", &len );
+            }
+            if ( len == 0 )
+            {
+                kdDebug() << "Z3950Connection::run() - giving up" << endl;
+                ZOOM_resultset_destroy( resultSet );
+                ZOOM_query_destroy( query );
+                done( i18n( "Record syntax error" ), MessageHandler::Error );
+                return;
+            }
+            kdDebug() << "Z3950Connection::run() - final syntax is " << newSyntax << endl;
+        }
+    }
+
+    // go back to fooling ourselves and calling it mods
+    if ( m_syntax == Latin1Literal( "xml" ) )
+    {
+        m_syntax = QString::fromLatin1( "mods" );
+    }
+    if ( newSyntax == Latin1Literal( "xml" ) )
+    {
+        newSyntax = QString::fromLatin1( "mods" );
+    }
+    // save syntax change for next time
+    if ( m_syntax != newSyntax )
+    {
+        qApp->postEvent( m_fetcher, new Z3950SyntaxChange( newSyntax ) );
+        m_syntax = newSyntax;
+    }
+
+    if ( m_sourceCharSet.isEmpty() )
+    {
+        m_sourceCharSet = QString::fromLatin1( "marc-8" );
+    }
+
+    const size_t realLimit = QMIN( numResults, m_limit );
+
+    for ( size_t i = m_start; i < realLimit && !m_aborted; ++i )
+    {
+        kdDebug() << "Z3950Connection::run() - grabbing index " << i << endl;
+        ZOOM_record rec = ZOOM_resultset_record( resultSet, i );
+        if ( !rec )
+        {
+            kdDebug() << "Z3950Connection::run() - no record returned for index " << i << endl;
+            continue;
+        }
+        int len;
+        QString data;
+        if ( m_syntax == Latin1Literal( "mods" ) )
+        {
+            data = toString( ZOOM_record_get( rec, "xml", &len ) );
+        }
+        else if ( m_syntax == Latin1Literal( "grs-1" ) )   // grs-1
+        {
+            // we're going to parse the rendered data, very ugly...
+            data = toString( ZOOM_record_get( rec, "render", &len ) );
+        }
+        else
+        {
+#if 0
+            kdWarning() << "Remove debug from z3950connection.cpp" << endl;
+            {
+                QFile f1( QString::fromLatin1( "/tmp/z3950.raw" ) );
+                if ( f1.open( IO_WriteOnly ) )
+                {
+                    QDataStream t( &f1 );
+                    t << ZOOM_record_get( rec, "raw", &len );
+                }
+                f1.close();
+            }
+#endif
+            data = toXML( ZOOM_record_get( rec, "raw", &len ), m_sourceCharSet );
+        }
+        Z3950ResultFound* ev = new Z3950ResultFound( data );
+        QApplication::postEvent( m_fetcher, ev );
+    }
+
+    ZOOM_resultset_destroy( resultSet );
+    ZOOM_query_destroy( query );
+
+    m_hasMore = m_limit < numResults;
+    if ( m_hasMore )
+    {
+        m_start = m_limit;
+        m_limit += Z3950_DEFAULT_MAX_RECORDS;
+    }
+#endif
+    done();
+}
+
+bool Z3950Connection::makeConnection()
+{
+    if ( m_connected )
+    {
+        return true;
+    }
+//  kdDebug() << "Z3950Connection::makeConnection() - " << m_fetcher->source() << endl;
+// I don't know what to do except assume database, user, and password are in locale encoding
+#ifdef HAVE_YAZ
+    d->conn_opt = ZOOM_options_create();
+    ZOOM_options_set( d->conn_opt, "implementationName", "KBibTeX" );
+    ZOOM_options_set( d->conn_opt, "databaseName",       toCString( m_dbname ) );
+    ZOOM_options_set( d->conn_opt, "user",               toCString( m_user ) );
+    ZOOM_options_set( d->conn_opt, "password",           toCString( m_password ) );
+
+    d->conn = ZOOM_connection_create( d->conn_opt );
+    ZOOM_connection_connect( d->conn, m_host.latin1(), m_port );
+
+    int errcode;
+    const char* errmsg; // unused: carries same info as 'errcode'
+    const char* addinfo;
+    errcode = ZOOM_connection_error( d->conn, &errmsg, &addinfo );
+    if ( errcode != 0 )
+    {
+        ZOOM_options_destroy( d->conn_opt );
+        ZOOM_connection_destroy( d->conn );
+        m_connected = false;
+
+        QString s = i18n( "Connection error %1: %2" ).arg( errcode ).arg( toString( errmsg ) );
+        if ( !QCString( addinfo ).isEmpty() )
+        {
+            s += " (" + toString( addinfo ) + ")";
+        }
+        kdDebug() << "Z3950Connection::makeConnection() - " << s << endl;
+        done( s, MessageHandler::Error );
+        return false;
+    }
+#endif
+    m_connected = true;
+    return true;
+}
+
+void Z3950Connection::done()
+{
+    checkPendingEvents();
+    qApp->postEvent( m_fetcher, new Z3950ConnectionDone( m_hasMore ) );
+}
+
+void Z3950Connection::done( const QString& msg_, int type_ )
+{
+    checkPendingEvents();
+    if ( m_aborted )
+    {
+        qApp->postEvent( m_fetcher, new Z3950ConnectionDone( m_hasMore ) );
+    }
+    else
+    {
+        qApp->postEvent( m_fetcher, new Z3950ConnectionDone( m_hasMore, msg_, type_ ) );
+    }
+}
+
+void Z3950Connection::checkPendingEvents()
+{
+    // if there's still some pending result events, go ahead and just wait 1 second
+    if ( resultsLeft > 0 )
+    {
+        sleep( 1 );
+    }
+}
+
+inline
+QCString Z3950Connection::toCString( const QString& text_ )
+{
+    return iconvRun( text_.utf8(), QString::fromLatin1( "utf-8" ), m_sourceCharSet );
+}
+
+inline
+QString Z3950Connection::toString( const QCString& text_ )
+{
+    return QString::fromUtf8( iconvRun( text_, m_sourceCharSet, QString::fromLatin1( "utf-8" ) ) );
+}
+
+// static
+QCString Z3950Connection::iconvRun( const QCString& text_, const QString& fromCharSet_, const QString& toCharSet_ )
+{
+#ifdef HAVE_YAZ
+    if ( text_.isEmpty() )
+    {
+        return text_;
+    }
+
+    if ( fromCharSet_ == toCharSet_ )
+    {
+        return text_;
+    }
+
+    yaz_iconv_t cd = yaz_iconv_open( toCharSet_.latin1(), fromCharSet_.latin1() );
+    if ( !cd )
+    {
+        // maybe it's iso 5426, which we sorta support
+        QString charSetLower = fromCharSet_.lower();
+        charSetLower.remove( '-' ).remove( ' ' );
+        if ( charSetLower == Latin1Literal( "iso5426" ) )
+        {
+            return iconvRun( Iso5426Converter::toUtf8( text_ ).utf8(), QString::fromLatin1( "utf-8" ), toCharSet_ );
+        }
+        else if ( charSetLower == Latin1Literal( "iso6937" ) )
+        {
+            return iconvRun( Iso6937Converter::toUtf8( text_ ).utf8(), QString::fromLatin1( "utf-8" ), toCharSet_ );
+        }
+        kdWarning() << "Z3950Connection::iconvRun() - conversion from " << fromCharSet_
+        << " to " << toCharSet_ << " is unsupported" << endl;
+        return text_;
+    }
+
+    const char* input = text_;
+    size_t inlen = text_.length();
+
+    size_t outlen = 2 * inlen;  // this is enough, right?
+    QMemArray<char> result0( outlen );
+    char* result = result0.data();
+
+    int r = yaz_iconv( cd, const_cast<char**>( &input ), &inlen, &result, &outlen );
+    if ( r <= 0 )
+    {
+        kdDebug() << "Z3950Connection::iconvRun() - can't decode buffer" << endl;
+        return text_;
+    }
+    // bug in yaz, need to flush buffer to catch last character
+    yaz_iconv( cd, 0, 0, &result, &outlen );
+
+    // length is pointer difference
+    size_t len = result - result0;
+
+    QCString output = QCString( result0, len + 1 );
+//  kdDebug() << "-------------------------------------------" << endl;
+//  kdDebug() << output << endl;
+//  kdDebug() << "-------------------------------------------" << endl;
+    yaz_iconv_close( cd );
+    return output;
+#endif
+    return text_;
+}
+
+QString Z3950Connection::toXML( const QCString& marc_, const QString& charSet_ )
+{
+#ifdef HAVE_YAZ
+    if ( marc_.isEmpty() )
+    {
+        kdDebug() << "Z3950Connection::toXML() - empty string" << endl;
+        return QString::null;
+    }
+
+    yaz_iconv_t cd = yaz_iconv_open( "utf-8", charSet_.latin1() );
+    if ( !cd )
+    {
+        // maybe it's iso 5426, which we sorta support
+        QString charSetLower = charSet_.lower();
+        charSetLower.remove( '-' ).remove( ' ' );
+        if ( charSetLower == Latin1Literal( "iso5426" ) )
+        {
+            return toXML( Iso5426Converter::toUtf8( marc_ ).utf8(), QString::fromLatin1( "utf-8" ) );
+        }
+        else if ( charSetLower == Latin1Literal( "iso6937" ) )
+        {
+            return toXML( Iso6937Converter::toUtf8( marc_ ).utf8(), QString::fromLatin1( "utf-8" ) );
+        }
+        kdWarning() << "Z3950Connection::toXML() - conversion from " << charSet_ << " is unsupported" << endl;
+        return QString::null;
+    }
+
+    yaz_marc_t mt = yaz_marc_create();
+    yaz_marc_iconv( mt, cd );
+    yaz_marc_xml( mt, YAZ_MARC_MARCXML );
+
+    // first 5 bytes are length
+    bool ok;
+#if YAZ_VERSIONL < 0x030000
+    int len = marc_.left( 5 ).toInt( &ok );
+#else
+    size_t len = marc_.left( 5 ).toInt( &ok );
+#endif
+    if ( ok && ( len < 25 || len > 100000 ) )
+    {
+        kdDebug() << "Z3950Connection::toXML() - bad length: " << ( ok ? len : -1 ) << endl;
+        return QString::null;
+    }
+
+#if YAZ_VERSIONL < 0x030000
+    char* result;
+#else
+    const char* result;
+#endif
+    int r = yaz_marc_decode_buf( mt, marc_, -1, &result, &len );
+    if ( r <= 0 )
+    {
+        kdDebug() << "Z3950Connection::toXML() - can't decode buffer" << endl;
+        return QString::null;
+    }
+
+    QString output = QString::fromLatin1( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" );
+    output += QString::fromUtf8( QCString( result, len + 1 ), len + 1 );
+//  kdDebug() << QCString(result) << endl;
+//  kdDebug() << "-------------------------------------------" << endl;
+//  kdDebug() << output << endl;
+    yaz_iconv_close( cd );
+    yaz_marc_destroy( mt );
+
+    return output;
+#else // no yaz
+    return QString::null;
+#endif
+}