Changeset 1155

Timestamp:

01/07/06 15:15:26 (3 years ago)

Author:

sdcosta

Message:

Important: Please see these important notes before svn'ing up
past this changeset.

This merges the no-magpie-cache branch back into the trunk.
svn merge -r 1114:1154 http://svn.gregarius.net/svn/branches/no-magpie-cache

Location:

trunk/rss

Files:

• admin/markallasread.php (added)
• extlib/rss_dbcache.inc (copied) (copied from branches/no-magpie-cache/extlib/rss_dbcache.inc)
• extlib/rss_fetch.inc (modified) (3 diffs)
• schema.php (modified) (5 diffs)
• util.php (modified) (6 diffs)

trunk/rss/extlib/rss_fetch.inc

@@ -29 +29 @@
 
 require_once( MAGPIE_DIR . 'rss_parse.inc' );
-require_once( MAGPIE_DIR . 'rss_cache.inc' );
+require_once( MAGPIE_DIR . 'rss_dbcache.inc' ); //sameer: changed to use db
 
 // for including 3rd party libraries
@@ -131 +131 @@
         // 4. if remote fails, return stale object, or error
         
-        $cache = new RSSCache( MAGPIE_CACHE_DIR, MAGPIE_CACHE_AGE );
+    // sameer: changed to use db
+        $cache = new RSSdbCache( MAGPIE_CACHE_DIR, MAGPIE_CACHE_AGE );
         
         if (MAGPIE_DEBUG and $cache->ERROR) {
@@ -145 +146 @@
         // store parsed XML by desired output encoding
         // as character munging happens at parse time
-        $cache_key       = $url . MAGPIE_OUTPUT_ENCODING;
+        //$cache_key       = $url . MAGPIE_OUTPUT_ENCODING;
+    // sameer: removed the encoding from the key
+        $cache_key       = $url;
         
         if (!$cache->ERROR) {

trunk/rss/schema.php

@@ -117 +117 @@
             }
         break;
+        case 'c.daterefreshed':
+        case 'daterefreshed':
+            // date feed was last refreshed, added in 0.5.3
+            rss_query('alter table ' .getTable('channels') .' add column daterefreshed datetime null');
+            if (rss_is_sql_error(RSS_SQL_ERROR_NO_ERROR)) {
+                $updated++;
+                rss_error("updated schema for table " . getTable('channels'), RSS_ERROR_NOTICE);
+            } else {
+                rss_error("Failed updating schema for table " . getTable('channels')
+                .": " . rss_sql_error_message(), RSS_ERROR_ERROR
+                );
+            }
+        // break; - fallthrough allowed on purpose because these are added at the same time
+        case 'c.refreshinterval':
+        case 'refreshinterval':
+            // refresh interval of a feed (in minutes), added in 0.5.3
+            rss_query('alter table ' .getTable('channels') .' add column refreshinterval int(16) not null default 60');
+            if (rss_is_sql_error(RSS_SQL_ERROR_NO_ERROR)) {
+                $updated++;
+                rss_error("updated schema for table " . getTable('channels'), RSS_ERROR_NOTICE);
+            } else {
+                rss_error("Failed updating schema for table " . getTable('channels')
+                .": " . rss_sql_error_message(), RSS_ERROR_ERROR
+                );
+            }
+        // break; - fallthrough allowed on purpose because these are added at the same time
+        case 'c.etag':
+        case 'etag':
+            // etag of the feed, (from HTTP header) added in 0.5.3
+            rss_query('alter table ' .getTable('channels') .' add column etag varchar(255) default null');
+            if (rss_is_sql_error(RSS_SQL_ERROR_NO_ERROR)) {
+                $updated++;
+                rss_error("updated schema for table " . getTable('channels'), RSS_ERROR_NOTICE);
+            } else {
+                rss_error("Failed updating schema for table " . getTable('channels')
+                .": " . rss_sql_error_message(), RSS_ERROR_ERROR
+                );
+            }
+        // break; - fallthrough allowed on purpose because these are added at the same time
+        case 'c.lastmodified':
+        case 'lastmodified':
+            // last modified code returned by the feed (from HTTP header), added in 0.5.3
+            rss_query('alter table ' .getTable('channels') .' add column lastmodified varchar(255) default null');
+            if (rss_is_sql_error(RSS_SQL_ERROR_NO_ERROR)) {
+                $updated++;
+                rss_error("updated schema for table " . getTable('channels'), RSS_ERROR_NOTICE);
+            } else {
+                rss_error("Failed updating schema for table " . getTable('channels')
+                .": " . rss_sql_error_message(), RSS_ERROR_ERROR
+                );
+            }
+        break;
         case 'i.author':
         case 'author':
@@ -147 +199 @@
             // enclosure for an item
             rss_query('alter table ' . getTable('item') . ' add column enclosure varchar(255) null');
+            if (rss_is_sql_error(RSS_SQL_ERROR_NO_ERROR)) {
+                $updated++;
+                rss_error('updated schema for table ' . getTable('item'), RSS_ERROR_NOTICE);
+            } else {
+                rss_error('Failed updating schema for table ' . getTable('item') . ': '
+                    . rss_sql_error_message(), RSS_ERROR_ERROR);
+            }
+        break;
+        case 'i.md5sum':
+        case 'md5sum':
+            // md5check on an item - added in 0.5.3
+            rss_query('alter table ' . getTable('item') . ' add column md5sum varchar(32) null');
+            if (rss_is_sql_error(RSS_SQL_ERROR_NO_ERROR)) {
+                $updated++;
+                rss_error('updated schema for table ' . getTable('item'), RSS_ERROR_NOTICE);
+            } else {
+                rss_error('Failed updating schema for table ' . getTable('item') . ': '
+                    . rss_sql_error_message(), RSS_ERROR_ERROR);
+            }
+        // break; - fallthrough allowed on purpose because these are added at the same time
+        case 'i.guid':
+        case 'guid':
+            // guid of an item - added in 0.5.3
+            rss_query('alter table ' . getTable('item') . ' add column guid text null');
+            rss_query('alter table ' . getTable('item') . ' add index `guid` (`guid`(10))');
             if (rss_is_sql_error(RSS_SQL_ERROR_NO_ERROR)) {
                 $updated++;
@@ -174 +251 @@
             descr varchar(255) default NULL,
             dateadded datetime default NULL,
+            daterefreshed datetime default NULL,
+            refreshinterval int(16) NOT NULL default '60',
+            etag varchar(255) default NULL,
+            lastmodified varchar(255) default NULL,
             icon varchar(255) default NULL,
             position int(11) NOT NULL default '0',
             mode int(16) NOT NULL default '1',
-            PRIMARY KEY  (id)
+            PRIMARY KEY  (id),
+            KEY url (url)
         ) TYPE=MyISAM;    
 _SQL_
@@ -333 +415 @@
           id bigint(16) NOT NULL auto_increment,
           cid bigint(11) NOT NULL default '0',
+          md5sum varchar(32) default NULL,
+          guid text default NULL,
           added datetime NOT NULL default '0000-00-00 00:00:00',
           title varchar(255) default NULL,
@@ -343 +427 @@
           PRIMARY KEY  (id),
           KEY url (url),
+          KEY guid(guid(10)),
           KEY cid (cid)
         ) TYPE=MyISAM;    

trunk/rss/util.php

@@ -158 +158 @@
             return array (magpie_error(), array ());
         }
-        elseif (!$rss) {
-            continue;
+        elseif (!$rss || !($rss->rss_origin & MAGPIE_FEED_ORIGIN_HTTP_200) ) {
+            continue; // no need to do anything if we do not get a 200 OK from the feed
         }
 
@@ -175 +175 @@
             if(!isset($item))
                 continue;
-            // item title: strip out html tags, shouldn't supposed
-            // to have any, should it?
-            //$title = strip_tags($item['title']);
+
+            // item title: strip out html tags
             $title = array_key_exists('title', $item) ? strip_tags($item['title']) : "";
             //$title = str_replace('& ', '& ', $title);
+
+
+            $description = "";
             // item content, if any
             if (array_key_exists('content', $item) && is_array($item['content']) && array_key_exists('encoded', $item['content'])) {
-                $description = kses($item['content']['encoded'], $kses_allowed);
+                $description = $item['content']['encoded'];
             }
             elseif (array_key_exists('description', $item)) {
-                $description = kses($item['description'], $kses_allowed);
+                $description = $item['description'];
             }
             elseif (array_key_exists('atom_content', $item)) {
-                $description = kses($item['atom_content'], $kses_allowed);
+                $description = $item['atom_content'];
             }
             elseif (array_key_exists('summary', $item)) {
-                $description = kses($item['summary'], $kses_allowed);
+                $description = $item['summary'];
             }
             else {
@@ -197 +199 @@
             }
 
-            if ($description != "" && $baseUrl != "") {
-                //$description = make_abs($description, $baseUrl);
-                $description = relative_to_absolute($description, $baseUrl);
+            $md5sum = "";
+            $guid = "";
+
+            if(array_key_exists('guid', $item) && $item['guid'] != "") {
+                $guid = $item['guid'];
+            }
+            elseif(array_key_exists('id', $item) && $item['id'] != "") {
+                $guid = $item['id'];
             }
 
             if ($description != "") {
-                $description = rss_plugin_hook('rss.plugins.import.description', $description);
-            }
+                $md5sum = md5($description);
+                $description = kses($description, $kses_allowed); // strip out tags
+
+                if ($baseUrl != "") {
+                    $description = relative_to_absolute($description, $baseUrl);
+                }
+            }
+
+            // Now let plugins modify the description
+            $description = rss_plugin_hook('rss.plugins.import.description', $description);
+
 
             // link
@@ -272 +288 @@
                 continue;
             }
+
             $dbtitle = rss_real_escape_string($title);
             if (strlen($dbtitle) >= 255) {
                 $dbtitle=substr($dbtitle,0,254);
             }
-            // check wether we already have this item
-            $sql = "select id,length(description),unread from ".getTable("item")." where cid=$cid and url='$url' and title='$dbtitle'";
-            $subres = rss_query($sql);
-            list ($indb, $dbdesc_len, $state) = rss_fetch_row($subres);
 
             if ($cDate > 0) {
@@ -287 +300 @@
             }
 
-            if ($indb == "") {
-
-                list ($cid, $dbtitle, $url, $description)
-                = rss_plugin_hook('rss.plugins.items.new', array ($cid, $dbtitle, $url, $description));
+            // check whether we already have this item
+            if ($guid) {
+                $sql = "select id,unread, md5sum, guid, pubdate from ".getTable("item")
+                       ." where cid=$cid and guid='$guid'";
+            } else {
+                $sql = "select id,unread, md5sum, guid, pubdate from ".getTable("item")
+                       ." where cid=$cid and url='$url' and title='$dbtitle'"
+                       ." and (pubdate is NULL OR pubdate=$sec)";
+            }
+            $subres = rss_query($sql);
+            list ($indb, $state, $dbmd5sum, $dbGuid, $dbPubDate) = rss_fetch_row($subres);
+
+            if ($indb && !($state & RSS_MODE_DELETED_STATE) && $md5sum != $dbmd5sum) {
+                // the md5sums do not match.
+                if(getConfig('rss.input.allowupdates')) { // Are we allowed update items in the db?
+                    list ($cid, $indb, $description) =
+                        rss_plugin_hook('rss.plugins.items.updated', array ($cid, $indb, $description));
+
+                    $sql = "update ".getTable("item")
+                           ." set "." description='".rss_real_escape_string($description)."', "
+                           ." unread = unread | ".RSS_MODE_UNREAD_STATE
+                           .", md5sum='$md5sum'" . " where cid=$cid and id=$indb";
+
+                    rss_query($sql);
+                    $updatedIds[] = $indb;
+                    continue;
+                }
+            }
+
+            if ($indb == "") { // This must be new item then. In you go.
+
+                list ($cid, $dbtitle, $url, $description) =
+                    rss_plugin_hook('rss.plugins.items.new', array ($cid, $dbtitle, $url, $description));
 
                 $sql = "insert into ".getTable("item")
                        ." (cid, added, title, url, enclosure,"
-                       ." description, author, unread, pubdate) "
+                       ." description, author, unread, pubdate, md5sum, guid) "
                        ." values ("."$cid, now(), '$dbtitle', "
                        ." '$url', '".rss_real_escape_string($enclosure)."', '"
                        .rss_real_escape_string($description)."', '"
                        .rss_real_escape_string($author)."', "
-                       ."$mode, $sec)";
+                       ."$mode, $sec, '$md5sum', '$guid')";
 
                 rss_query($sql);
-
 
                 $newIid = rss_insert_id();
                 $updatedIds[] = $newIid;
-                /*
-                 * Ticket #26: Add hook to modify the item just after it 
-                 * has been inserted into the database
-                 */
                 rss_plugin_hook('rss.plugins.items.newiid',array($newIid,$item,$cid));
             }
-            elseif (!($state & RSS_MODE_DELETED_STATE) &&
-                    getConfig('rss.input.allowupdates') &&
-                    strlen($description) > $dbdesc_len) {
-
-                list ($cid, $indb, $description) =
-                    rss_plugin_hook('rss.plugins.items.updated', array ($cid, $indb, $description));
-
-                $sql = "update ".getTable("item")
-                       ." set "." description='".rss_real_escape_string($description)."', "
-                       ." unread = unread | ".RSS_MODE_UNREAD_STATE." where cid=$cid and id=$indb";
-
-                rss_query($sql);
-                $updatedIds[] = $indb;
-            }
+
         }
     }
@@ -557 +581 @@
     static $ret;
     if ($ret === NULL) {
-         $ret = dirname($_SERVER['PHP_SELF']);
-         if (defined('RSS_FILE_LOCATION') && eregi(RSS_FILE_LOCATION."\$", $ret)) {
-              $ret = substr($ret, 0, strlen($ret) - strlen(RSS_FILE_LOCATION));
-         }
-         if (substr($ret, -1) != "/") {
-              $ret .= "/";
-         }
+        $ret = dirname($_SERVER['PHP_SELF']);
+        if (defined('RSS_FILE_LOCATION') && eregi(RSS_FILE_LOCATION."\$", $ret)) {
+            $ret = substr($ret, 0, strlen($ret) - strlen(RSS_FILE_LOCATION));
+        }
+        if (substr($ret, -1) != "/") {
+            $ret .= "/";
+        }
     }
     return $ret;
-    
+
 }
 $dummy = getPath();