ó
‚•oTc           @   s×  d  d l  Z  d  d l Z d  d l Z d  d l Z d  d l Z d  d l Z d  d l m Z d  d l Z y e j	 d Z
 Wn d Z
 n Xd  d l Z d  d l j Z d  d l j Z d  d l m Z d  d l j Z dz Z d	 e f d
 „  ƒ  YZ e j d e j ƒ Z e j d e j ƒ Z e j d e j ƒ Z e j d e j ƒ Z d „  Z d „  Z e j d e j ƒ Z d „  Z d{ Z  d| Z! d} Z" d „  Z# i- d d  6d! d" 6d# d$ 6d% d& 6d' d( 6d) d* 6d+ d, 6d- d. 6d/ d0 6d1 d2 6d3 d4 6d5 d6 6d7 d8 6d9 d: 6d; d< 6d= d> 6d? d@ 6dA dB 6dC dD 6dE dF 6dG dH 6dI dJ 6dK dL 6dM dN 6dO dP 6dQ dR 6dS dT 6dU dV 6dW dX 6dY dZ 6d[ d\ 6d] d^ 6d_ d` 6da db 6dc dd 6de df 6dg dh 6di dj 6dk dl 6dm dn 6do dp 6dq dr 6ds dt 6du dv 6dw dx 6Z$ dy „  Z% d S(~   iÿÿÿÿN(   t   Threadt   httpst   http(   t   NzbQueues   .zips   nzbsa.co.zas   newshost.za.nett
   URLGrabberc           B   s>   e  Z d Z d  „  Z d d „ Z d „  Z d „  Z d „  Z RS(   c         C   sn   t  j |  ƒ t j ƒ  |  _ x9 t j j ƒ  D]( } | \ } } |  j j | | f ƒ q, Wt |  _	 |  t
 _ d  S(   N(   R    t   __init__t   Queuet   queueR   t   dot   get_urlst   putt   Falset   shutdownR   (   t   selft   tupt   urlt   nzo(    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyR   2   s    	c         C   s<   | r" | r" t  j  ƒ  | | _ n  |  j j | | f ƒ d  S(   N(   t   timet   waitR   R
   (   R   R   t
   future_nzot   when(    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyt   add;   s    c         C   sA   d | k r= t  j ƒ  r= | j d d ƒ } |  j | d  ƒ n  d  S(   Ns   nzbmatrix.coms   download.php?s   bookmarks.php?action=remove&(   t   cfgt   matrix_del_bookmarkt   replaceR   t   None(   R   R   (    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyt   rm_bookmarkA   s    c         C   s*   t  j d ƒ t |  _ |  j d  d  ƒ d  S(   Ns   URLGrabber shutting down(   t   loggingt   infot   TrueR   R   R   (   R   (    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyt   stopG   s    	c         C   sN  t  j d ƒ t |  _ x1|  j sIt j d ƒ |  j j ƒ  \ } } | sP q n  | rŠ | j rŠ | j t j ƒ  k rŠ |  j	 | | ƒ q n  | j
 d d ƒ } yz| } | rò y | j } Wn t k
 rÒ t } n X| rò t  j d | ƒ w qò n  t | ƒ \ } } | rt  j d | ƒ n t  j d | ƒ d | k rKt j i  ƒ } n t j i  ƒ } d  | _ g  | _ | j d	 d
 t j j ƒ g  t D] } | | k rt ^ qs¾| j d d ƒ n  d  } d  }	 d }
 i  } d } y | j | ƒ \ } } Wn d  } n X| rÀx²| j ƒ  D]¡} y$ | d j ƒ  } | d j ƒ  } Wn
 qn X| d= k rd| }	 n| d> k r}| | d <nê | d? k r®| } | j d ƒ sg| d 7} qgn¹ | d k rÇ| | d <n  | d k rà| | d <n‡ | d k rù| | d <nn | d k r| | d <nU | d k r+| | d <n< | d@ k rIt  j! | ƒ }
 n | d  k rgt  j! | ƒ } n  | sxI | D]> } d! | k rt| | j" d! ƒ d" j d# ƒ j d$ ƒ } qtqtWqqWn  | rst# | | ƒ \ } } } } | sU| r#t  j | ƒ t  j d% | | ƒ |  j	 | | | ƒ n, t  j$ | ƒ t  j% | t& | ƒ | d& t ƒw n  t' | |	 ƒ }	 | rúw qún‡ | rŽd } t } d  } n t( | | ƒ \ } } } } | sú| rÛt  j d' | ƒ |  j	 | | | ƒ n t  j% | | | d& t ƒw n  | st) j* j+ | ƒ d } n  | j, } | j- } | j. } | d  k sL| d( k rd|	 rdt  j/ |	 ƒ } n  | j0 } | j1 } t) j* j2 | ƒ d j ƒ  dA k rt3 j4 | | d* | d+ | d, | d- | d. | d/ | d0 | j5 ƒ\ } } | d k rt6 j7 j8 | j9 d1 t ƒq| d2 k r*t  j d3 | ƒ d4 } nA | d5 k rUt6 j7 j8 | j9 d1 t ƒw n t  j d6 | ƒ d7 } |  j	 | | | ƒ n” t3 j: | | | | | d- | d. | d0 | j5 ƒd d k r×t6 j7 j8 | j9 d1 t ƒn> y t) j8 | ƒ Wn n Xt  j d8 | ƒ |  j	 | | d9 ƒ Wq t  j$ d: d; t ƒt  j d< d; t ƒq Xq Wd  S(B   Ns   URLGrabber starting upg      @t    t    s"   Dropping URL %s, job entry missings   Removing nzbmatrix bookmark %ss   Grabbing URL %ss	   .nzbsrus.s
   User-Agents   SABnzbd+/%ss   Accept-encodingt   gzipi    i   t   category_ids   x-dnzb-categorys   x-dnzb-moreinfot	   more_infos   x-dnzb-names   .nzbs   x-dnzb-propernamet
   propernames   x-dnzb-episodenamet   episodenames   x-dnzb-yeart   years   x-dnzb-failuret   failures   x-dnzb-detailst   detailss   content-lengths   retry-afters	   filename=i	   t   ;t   "s,   Retry nzbmatrix item %s after waiting %s sect   retrys   Retry URL %st   *s   .gzt   ppt   scriptt   catt   priorityt   nzbnamet   nzo_infoR   t   add_to_historyiþÿÿÿs$   Incomplete NZB, retry after 5 min %si,  iÿÿÿÿs0   Unknown error fetching NZB, retry after 2 min %six   s6   Unknown filetype when fetching NZB, retry after 30s %si   s   URLGRABBER CRASHEDt   exc_infos   URLGRABBER Traceback: (   R"   s   x-dnzb-category(   s   x-dnzb-moreinfo(   s   x-dnzb-name(   s   content-length(   s   .nzbs   .gz(;   R   R   R   R   R   t   sleepR   t   getR   R   R   t   deletedt   AttributeErrorR   t   debugt   _matrix_urlt   urllibt	   URLopenert   FancyURLopenerR   t   prompt_user_passwdt
   addheaderst	   addheadert   sabnzbdt   versiont   __version__t   _BAD_GZ_HOSTSt   retrievet   itemst   lowert   stript   endswitht   misct   int_convt   indext   _analyse_matrixt   errort	   bad_fetcht   clean_matrix_urlt   get_matrix_categoryt   _analyse_otherst   ost   patht   basenameR-   R.   R/   t   cat_convertR0   t   custom_namet   splitextt
   dirscannert   ProcessSingleFileR   R   R   t   removet   nzo_idt   ProcessArchiveFile(   R   R   R   t   del_bookmarkR7   t	   matrix_idt   openert   itemt   filenamet   categoryt   lengthR2   R   t   fnt   headerR   t   valuet   msgR+   R-   R.   R/   R0   R1   t   rest   nzo_idsR   (    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyt   runL   s   	$
			%
	:							"'	N(	   t   __name__t
   __module__R   R   R   R   R   R   Rk   (    (    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyR   /   s   				s    nzbmatrix\.com/(.*)[\?&]id=(\d+)s   nzbxxx\.com/(.*)[\?&]id=(\d+)s   &username=([^&=]+)s   &apikey=([^&=]+)c         C   s  d } t  j |  ƒ } | s- t j |  ƒ } n  | rf d } t j t j ƒ  ƒ } t j t j ƒ  ƒ } n? | r¥ d } t j t j ƒ  ƒ } t j t j	 ƒ  ƒ } | } n  | rù | j
 d ƒ } t j |  ƒ sÚ t j |  ƒ rù d t | | | | f }  qù n  |  | f S(   Ni    s   nzbmatrix.coms
   nzbxxx.comi   s9   %s://api.%s/v1.1/download.php?id=%s&username=%s&apikey=%s(   t   _RE_NZBMATRIXt   searcht
   _RE_NZBXXXR;   t
   quote_plusR   t   matrix_usernamet   matrix_apikeyt   xxx_usernamet
   xxx_apikeyt   groupt   _RE_NZBMATRIX_USERt   _RE_NZBMATRIX_APIt	   _PROTOCOL(   R   R_   t   mt   mxt   sitet   usert   key(    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyR:     s&    	 c         C   sb   d } t  j |  ƒ } | s3 t j |  ƒ } d } n  | r^ | j d ƒ } d t | | f }  n  |  S(   Ns   nzbmatrix.coms
   nzbxxx.comi   s#   %s://api.%s/v1.1/download.php?id=%s(   Rn   Ro   Rp   Rv   Ry   (   R   R|   Rz   R_   (    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyRP     s    	s   please_wait[_ ]+(\d+)c         C   sõ  d } d } |  s/ t  j d ƒ d  | t d f Sy2 t |  d ƒ } | j d ƒ j ƒ  } | j ƒ  Wn$ t  j d |  ƒ d  | t d f SX| r³ d | k r³ t d	 ƒ | } d
 } n| rÏ| j	 d ƒ rÏt
 j | d ƒ } | rd | k rû t d ƒ } n t d ƒ d | } d  | t d f Sd | k r:d } d } qÏt
 j | d ƒ r[d } d } qÏd | k r€t d	 ƒ | } d
 } qÏt j | ƒ } | r¶t t | j d ƒ ƒ d ƒ } qÏt d ƒ | } d } n  | råd  | t | f S|  | t d f S(    NR    i    s.   No response from nzbmatrix, retry after 60 seci<   t   ri(   s;   Problem with tempfile %s from nzbmatrix, retry after 60 secs	   <!DOCTYPEs"   Invalid nzbmatrix report number %si,  RN   t   invalid_logint   invalid_apit   disabled_accountt   vip_onlyt   vips0   You need an nzbmatrix VIP account to use the APIs   Invalid nzbmatrix credentialss    (%s)t   limit_reacheds'   Too many nzbmatrix hits, waiting 10 miniX  t   daily_limits   limit is reacheds-   Daily limit nzbmatrix reached, waiting 1 houri  t   no_nzb_foundi   s'   Problem accessing nzbmatrix server (%s)(   R€   R   R‚   Rƒ   (   R†   s   limit is reached(   R   R9   R   R   t   opent   readRG   t   closet   Tat
   startswithRJ   t	   match_strR   t   _RE_MATRIX_ERRRo   t   mint   intRv   (   Re   R_   Rh   R   t   ft   datat   txtRz   (    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyRM   .  sN    				!	t   DENIED_MISSING_CREDENTIALSt   DENIED_NO_ACCOUNTt   DENIED_INVALID_CREDENTIALSt   INCORRECT_URLt   NZB_DELETEDt
   POST_NUKEDt   FILE_UNAVAILABLEt	   SQL_ERRORt   SERVICE_OFFLINEt#   MAX_DOWNLOAD_REACHED_UPGRADE_TO_VIPt   MAX_DOWNLOAD_REACHEDc         C   sZ  d } d } |  s/ t  j d ƒ d  | t d f Sy, t |  d ƒ } | j d ƒ } | j ƒ  Wn$ t  j d |  ƒ d  | t d f SX| s¥ t  j d ƒ d  | t d f Sd	 | k rJt j | t	 ƒ rã t  j d
 | ƒ d  | t
 d f St j | t ƒ rt  j d | ƒ d  | t d f St j | t ƒ rJt  j d | ƒ d  | t d f Sn  |  | t
 d f S(   NR    i    s,   No response from indexer, retry after 60 seci<   R   id   s9   Problem with tempfile %s from indexer, retry after 60 secs1   Received nothing from indexer, retry after 60 secs	   .nzbsrus.s   nzbsrus says: %s, aborts   nzbsrus says: %s, wait 15mi„  s   nzbsrus says: %s, wait 60mi  (   R   R9   R   R   Rˆ   R‰   RŠ   RJ   R   t	   RUS_FATALR   t   RUS_15Mt   RUS_60M(   Re   R   Rh   R   R‘   R’   (    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyRR   m  s4    s	   anime.allt   28s
   apps.linuxt   20s   apps.mact   19s
   apps.othert   21s   apps.pct   18s   apps.portablet   52s   documentaries.hdt   53s   documentaries.stdt   9s   games.dst   45s   games.othert   17s   games.pct   10s	   games.ps2t   11s	   games.ps3t   43s	   games.pspt   12s	   games.wiit   44s   games.wii vct   51s
   games.xboxt   13s   games.xbox360t   14s   games.xbox360 (other)t   56s   movies.sd (image)t   1s	   movies.sdt   2s   movies.hd (remux)t   54s   movies.hd (x264)t   42s   movies.hd (image)t   50s   movies.othert   4s   music.sd (image)t   24s   music.losslesst   23s   music.mp3, albumst   22s   music.mp3, singlest   47s   music.othert   27s   music.videot   25s   other.androidt   55s   other.audio, bookst   49s   other.e-bookst   36s   other.extra, pars/fillst   39s   other.imagest   37s   other.iOS/iPhonet   38s   other.othert   40s   other.radiot   26s   tv.sd (image)t   5s   tv.hd (image)t   57s   tv.hd (x264)t   41s   tv.othert   8s   tv.sdt   6s   tv.sport/entt   7c         C   s4   t  j | | ƒ } | r, d |  k r, d | S| Sd  S(   Ns
   nzbxxx.coms   XXX: (   t   _MATRIX_MAPR6   (   R   Rc   (    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyRQ   Â  s    (   s   .zips   nzbsa.co.zas   newshost.za.net(   R”   R•   R–   R—   R˜   R™   Rš   (   R›   Rœ   (   R   Rž   (&   RS   R   t   reR   R   R;   t	   threadingR    t   sockett   sslRy   RA   t   sabnzbd.miscRJ   t   sabnzbd.dirscannerRY   t   sabnzbd.nzbqueueR   t   sabnzbd.cfgR   RD   R   t   compilet   IRn   Rp   Rw   Rx   R:   RP   RŽ   RM   RŸ   R    R¡   RR   RÏ   RQ   (    (    (    s&   /var/www/SABnzbd/sabnzbd/urlgrabber.pyt   <module>   s    

Ð			8  	%
