From c7656971a2a9e0e6da20e88faf4ba39e88d5bc6a Mon Sep 17 00:00:00 2001 From: KZ Date: Tue, 25 Aug 2015 21:30:32 +0100 Subject: [PATCH] Implement animetorrents #150 --- README.md | 193 +-- src/Jackett/Content/logos/animetorrents.png | Bin 0 -> 8409 bytes src/Jackett/Indexers/AnimeTorrents.cs | 176 +++ src/Jackett/Indexers/BaseIndexer.cs | 903 +++++++------- src/Jackett/Jackett.csproj | 1208 ++++++++++--------- src/Jackett/JackettModule.cs | 163 +-- src/Jackett/Utils/Clients/HttpWebClient.cs | 228 ++-- src/Jackett/Utils/Clients/WebRequest.cs | 184 +-- 8 files changed, 1626 insertions(+), 1429 deletions(-) create mode 100644 src/Jackett/Content/logos/animetorrents.png create mode 100644 src/Jackett/Indexers/AnimeTorrents.cs diff --git a/README.md b/README.md index 8c38e4ce8..cc8422dd4 100644 --- a/README.md +++ b/README.md @@ -1,97 +1,98 @@ -## Jackett - -#### Download -Downloads on the [Releases page](https://github.com/zone117x/Jackett/releases) - - -#### Overview -This software creates a [Torznab](https://github.com/Sonarr/Sonarr/wiki/Implementing-a-Torznab-indexer) (with [nZEDb](https://github.com/nZEDb/nZEDb/blob/master/docs/newznab_api_specification.txt) category numbering) and [TorrentPotato](https://github.com/RuudBurger/CouchPotatoServer/wiki/Couchpotato-torrent-provider) API server on your machine. Torznab enables software such as [Sonarr](https://sonarr.tv) to access data from your favorite indexers in a similar fashion to rss but with added features such as searching. TorrentPotato is an interface accessible to [CouchPotato](https://couchpota.to/). - -Jackett works as a proxy server: it translates queries from apps (Sonarr, SickRage, CouchPotato, Mylar, etc) into tracker-site-specific http queries, parses the html response, then sends results back to the requesting software. This allows for getting recent uploads (like RSS) and performing searches. Jackett is a single repository of maintained indexer scraping & translation logic - removing the burden from other apps. - -We were previously focused on TV but are working on extending searches to allow for searching other items such as movies, comics, and music. - - -#### Supported Systems -* Windows using .NET 4.5 -* Linux and OSX using Mono 4 (v3 should work but you may experience crashes). - - -#### Supported Trackers - * [AlphaRatio](https://alpharatio.cc/) - * [AnimeBytes](https://animebytes.tv/) - * [Avistaz](https://avistaz.to/) - * [BakaBT](http://bakabt.me/) - * [bB](http://reddit.com/r/baconbits) - * [BeyondHD](https://beyondhd.me/) - * [BIT-HDTV](https://www.bit-hdtv.com) - * [BitMeTV](http://www.bitmetv.org/) - * [Demonoid](http://www.demonoid.pw/) - * [EuTorrents](https://eutorrents.to/) - * [FileList](http://filelist.ro/) - * [FrenchTorrentDb](http://www.frenchtorrentdb.com/) - * [Freshon](https://freshon.tv/) - * [HD-Space](https://hd-space.org/) - * [HD-Torrents.org](https://hd-torrents.org/) - * [Immortalseed.me](http://immortalseed.me) - * [IPTorrents](https://iptorrents.com/) - * [MoreThan.tv](https://morethan.tv/) - * [pretome](https://pretome.info) - * [PrivateHD](https://privatehd.to/) - * [RARGB](https://rarbg.to/) - * [RuTor](http://rutor.org/) - * [SceneAccess](https://sceneaccess.eu/login) - * [SceneTime](https://www.scenetime.com/) - * [ShowRSS](https://showrss.info/) - * [Strike](https://getstrike.net/) - * [T411](http://www.t411.io/) - * [The Pirate Bay](https://thepiratebay.se/) - * [TorrentBytes](https://www.torrentbytes.net/) - * [TorrentDay](https://torrentday.eu/) - * [TorrentLeech](http://www.torrentleech.org/) - * [TorrentShack](http://torrentshack.me/) - * [Torrentz](https://torrentz.eu/) - * [TV Chaos UK](https://tvchaosuk.com/) - -#### Installation on Linux/OSX - 1. Install [Mono 4](http://www.mono-project.com/download/) or better - 2. Install libcurl: - * Debian/Ubunutu: apt-get install libcurl-dev - * Redhat/Fedora: yum install libcurl-devel - * For other distros see the [Curl docs](http://curl.haxx.se/dlwiz/?type=devel). - 3. Download and extract the latest ```.tar.bz2``` release from the [website](http://jackett.net/Download) and run Jackett using mono with the command "mono JackettConsole.exe". - - -#### Installation on Windows - -Grab the latest release from the [web site](http://jackett.net/Download). - -We recommend you install Jackett as a Windows service using the supplied installer. When installed as a service the tray icon acts as a way to open/start/stop Jackett. If you opted to not install it as a service then Jackett will run its web server from the tray tool. - -Jackett can also be run from the command line using JackettConsole.exe if you would like to see log messages (Ensure the server isn't already running from the tray/service). - - -#### Troubleshooting - -* Command line switches - -You can pass various options when running via the command line, see --help for details. - -* Unable to connect to certain trackers on Linux - -Try running with the "--SSLFix true" if you are on Redhat/Fedora/NNS based libcurl. If the tracker is currently configured try removing it and adding it again. Alternatively try running with a different client via --UseClient (Warning: safecurl just executes curl and your details may be seen from the process list). - -* Enable logging - -You can get additional logging with the switches "-t -l". Please post logs if you are unable to resolve your issue with these switches ensuring to remove your username/password/cookies. - - -### Additional Trackers -Jackett's framework allows our team (and any other volunteering dev) to implement new trackers in an hour or two. If you'd like support for a new tracker then feel free to leave a request on the [issues page](https://github.com/zone117x/Jackett/issues) or contact us on IRC (see below). Pull requests must be made to the develop branch. - -### Contact & Support -Use the github issues pages or talk to us directly at: [irc.freenode.net#jackett](http://webchat.freenode.net/?channels=#jackett). - -### Screenshots - +## Jackett + +#### Download +Downloads on the [Releases page](https://github.com/zone117x/Jackett/releases) + + +#### Overview +This software creates a [Torznab](https://github.com/Sonarr/Sonarr/wiki/Implementing-a-Torznab-indexer) (with [nZEDb](https://github.com/nZEDb/nZEDb/blob/master/docs/newznab_api_specification.txt) category numbering) and [TorrentPotato](https://github.com/RuudBurger/CouchPotatoServer/wiki/Couchpotato-torrent-provider) API server on your machine. Torznab enables software such as [Sonarr](https://sonarr.tv) to access data from your favorite indexers in a similar fashion to rss but with added features such as searching. TorrentPotato is an interface accessible to [CouchPotato](https://couchpota.to/). + +Jackett works as a proxy server: it translates queries from apps (Sonarr, SickRage, CouchPotato, Mylar, etc) into tracker-site-specific http queries, parses the html response, then sends results back to the requesting software. This allows for getting recent uploads (like RSS) and performing searches. Jackett is a single repository of maintained indexer scraping & translation logic - removing the burden from other apps. + +We were previously focused on TV but are working on extending searches to allow for searching other items such as movies, comics, and music. + + +#### Supported Systems +* Windows using .NET 4.5 +* Linux and OSX using Mono 4 (v3 should work but you may experience crashes). + + +#### Supported Trackers + * [AlphaRatio](https://alpharatio.cc/) + * [AnimeBytes](https://animebytes.tv/) + * [AnimeTorrents](http://animetorrents.me/) + * [Avistaz](https://avistaz.to/) + * [BakaBT](http://bakabt.me/) + * [bB](http://reddit.com/r/baconbits) + * [BeyondHD](https://beyondhd.me/) + * [BIT-HDTV](https://www.bit-hdtv.com) + * [BitMeTV](http://www.bitmetv.org/) + * [Demonoid](http://www.demonoid.pw/) + * [EuTorrents](https://eutorrents.to/) + * [FileList](http://filelist.ro/) + * [FrenchTorrentDb](http://www.frenchtorrentdb.com/) + * [Freshon](https://freshon.tv/) + * [HD-Space](https://hd-space.org/) + * [HD-Torrents.org](https://hd-torrents.org/) + * [Immortalseed.me](http://immortalseed.me) + * [IPTorrents](https://iptorrents.com/) + * [MoreThan.tv](https://morethan.tv/) + * [pretome](https://pretome.info) + * [PrivateHD](https://privatehd.to/) + * [RARGB](https://rarbg.to/) + * [RuTor](http://rutor.org/) + * [SceneAccess](https://sceneaccess.eu/login) + * [SceneTime](https://www.scenetime.com/) + * [ShowRSS](https://showrss.info/) + * [Strike](https://getstrike.net/) + * [T411](http://www.t411.io/) + * [The Pirate Bay](https://thepiratebay.se/) + * [TorrentBytes](https://www.torrentbytes.net/) + * [TorrentDay](https://torrentday.eu/) + * [TorrentLeech](http://www.torrentleech.org/) + * [TorrentShack](http://torrentshack.me/) + * [Torrentz](https://torrentz.eu/) + * [TV Chaos UK](https://tvchaosuk.com/) + +#### Installation on Linux/OSX + 1. Install [Mono 4](http://www.mono-project.com/download/) or better + 2. Install libcurl: + * Debian/Ubunutu: apt-get install libcurl-dev + * Redhat/Fedora: yum install libcurl-devel + * For other distros see the [Curl docs](http://curl.haxx.se/dlwiz/?type=devel). + 3. Download and extract the latest ```.tar.bz2``` release from the [website](http://jackett.net/Download) and run Jackett using mono with the command "mono JackettConsole.exe". + + +#### Installation on Windows + +Grab the latest release from the [web site](http://jackett.net/Download). + +We recommend you install Jackett as a Windows service using the supplied installer. When installed as a service the tray icon acts as a way to open/start/stop Jackett. If you opted to not install it as a service then Jackett will run its web server from the tray tool. + +Jackett can also be run from the command line using JackettConsole.exe if you would like to see log messages (Ensure the server isn't already running from the tray/service). + + +#### Troubleshooting + +* Command line switches + +You can pass various options when running via the command line, see --help for details. + +* Unable to connect to certain trackers on Linux + +Try running with the "--SSLFix true" if you are on Redhat/Fedora/NNS based libcurl. If the tracker is currently configured try removing it and adding it again. Alternatively try running with a different client via --UseClient (Warning: safecurl just executes curl and your details may be seen from the process list). + +* Enable logging + +You can get additional logging with the switches "-t -l". Please post logs if you are unable to resolve your issue with these switches ensuring to remove your username/password/cookies. + + +### Additional Trackers +Jackett's framework allows our team (and any other volunteering dev) to implement new trackers in an hour or two. If you'd like support for a new tracker then feel free to leave a request on the [issues page](https://github.com/zone117x/Jackett/issues) or contact us on IRC (see below). Pull requests must be made to the develop branch. + +### Contact & Support +Use the github issues pages or talk to us directly at: [irc.freenode.net#jackett](http://webchat.freenode.net/?channels=#jackett). + +### Screenshots + ![screenshot](http://i.imgur.com/t1sVva6.png "screenshot") \ No newline at end of file diff --git a/src/Jackett/Content/logos/animetorrents.png b/src/Jackett/Content/logos/animetorrents.png new file mode 100644 index 0000000000000000000000000000000000000000..24c836e4b073bd8d0af47a33b90dc5d8a822e2ab GIT binary patch literal 8409 zcmbVSRZ|>HuwC3W=;CgRLvYss!6mqR2#W;>u(-RsemDdVfhEB`u(-Ph2<|L)xexab z+^Op6s;-&pdFY-#r)Q!y)s=8CftUaQ0Oy0Uy!M-}d=qhW#J4$M-LCab(ON5Oe*^%& zF#!NUp#Z?c>zh6V0K9krfFlb4Kr{;gAazYKA5eT-ivd`2>w?P0&UEb-;89Q z3ZFc6T`4IacFQe;Yc4CC4tNS5Sv0+3+ zH6t&lw#Yhfj@N!xOKFI*n^IAMmxH|sv81R{IfW6CK1CmtmTG%73$SxEF`|GwKv_s$aUVA*4B-uvyCXF7LE19!}Na!ctW?c zlT&p~O|$t+nqngL_p-79h$X>%ld%b0^7?ns!sO)S{JdS)6AI!v@@EbX4z8m>q-Unh z^z`(y($cUO8G3s9>FH^9cJ|{fsid5ooU$^e$Gv)j47J?%#~HkZ!~7{gLVSEpZEY*=do%>uum8C5X$yHQd!HVz zKB}oLnwr30u3a|w_Vy6i3&Uy`_3ayTZA*kyIOVEAp*5g0y-%p=#;B57w_JpsDID9Q z_^oSVR*dnPnVCpkjg5^73Am20A_$jNIu&*1UDTaaiwx7vXl>Weht8W@-AW0RAKnoN zc}1r|P1TD}$BAy(i_!GKnm@0()31k#+w+)?bJ&jaihVf^R$v(CSXfx0eIY1V=rvMw z(g_@$)#_|Jh8-BfP*;84ceoPpb?VjViwFyYY0=^-gidQ} z3sg69UogaFOnrZL==prtThytU1lA<6ySZX|X-&xTP9VNN@>U%Shz&O->xCWml1|UA z@drcO4O$;=K>QjJ{bAwY$Il6`DsVV_<)>?x&&9R@7ghc3?d>rs*RyFqDm68A=Q)rbEv}i}Jhk@uQ0)}xNoA?&9D^mXVZ$SPA}-zJ9&dwk6S!1R<(ail&Y)>0r!)_UDQWz2>g*Dkv!2f5pOzb%pfGRn*jQ02?rz zo&V85?q%fULb2&6y8k6gy+m1^O*{v;w?Dx0#q7tjdLXYMo?c#-;P>y}Z#;+88&fP{o0OlxavVMsRJ|BmKrMr*(F&(=`akHkjRetPkVN@QAmkY6Zv0US4DX_{Qe&QJT5M7 zGPSJaTc$T7?7Y3J>$Ka|aJ1U7x1`FS;lf5)`S!;TPP1QRhkuuv?K`r@NLe%<9v^qS zE_X*h<#Ms#j@_NMx~yoXk7e^$)YqTMsjJ852V}$;L9>wW-**3wkCk%U?(YnE1blV2 z&^)YNA{$T2_&}PeMw%H^NmDpaKh)CF0%B!7aQO@ZzRE!>g$ZcA7^V81H9m53at`c8 zj5TyU-ksqEz^|W%r#^e(oR8@0R{=W=gZ1t`(q9n-QC{ywianS_?+(y<+yy!#{7>~Np-mD$H)k{Yw1& z1pP>~)0SxWwQ|;N%Io))R#lDZc)_Oeq7+XDPj@fqfbrx^x1HIYOt8C)M~2=w;n82{?Ir5Rz`2nHeR{7 zaQQtlK+*B>=@}V=u1k3$Hoojquhx(kGyx-?c#?}54IaT&7~;Ekd@9WHGxrP*E%-pr z*WD50pI^2@I&m?As~-N0>PLkCSeVyGLzB|i>D%>0=*e(ia2s$-+TGo)OG)|6RAvKt z!G{F15~ur+$VIdX&&E$8;awbW_IO$*e>&!3^D@`u3D2R`6!qqUEiNo$GU1aGg2BK* zR`u$V5IH)A2sfgoMGM*3Oanzl)CW{d@0-J!3T(FvCF}2Qq2$xkHX>U={IUiJ0!H3C zGsjEK9PI4NsVRJWd#U#iXIsZW;D`XEWs8`Ih)6ila%*dk6yFlXOS^f*cb{U-&i-t8 zM-6|9yVS{#hqkexGdM3Vuc&Bp-$g8+P=$$-p)66VbhSfO9P4IrilrotvNuvP5VkNA zz{;3K$2be5@Y@|GU}9nt_61v@V`wF29aNMv+uPd z8Hpr)r)##Y##oRO(Pj|N9jn=SLqaU)=MABviA+p9VY%0rd#bdU!Yl9I@iXEz_-cRrNawLfFj_NrvK;+2 zsFh^-Es@UFfB*Bp1u4jf(Se?PEjD#L_}juA9QS?OtPvwOC&0*`GnqIQTv*trvvVnc zz?djhgD_57;|t!Q;zXLAaAbzC@DdWvey+xua!VIKKR;`01}iIleSIYrOaV$pwyBFJ zkuenAFZQHAY%mhI>FIkPnqdFk_F+?PXp7X#x+RLI{Elt2!|brB^xMyBk{qF{R5e!~ z98B{latN!2&gaMR-B?=&t@ewoPqOJBX$rrd(!S&)v|_$cTsL=>q=r;1P8FUX)!<4f%?IZ6xE@uV2pdwTE1` zDSP91^k2``amFwY70C&$J|&Da#Ep%O(euv?l@Q5>p>{ywE6b9C?{R;f6`7%-6{8>P zN!shDl!z}|Sy@>m#NPNiu$t>I7->^_B(YlPNJ>h|$jI2>H6TR@|KPTM5$73Mn3jLk z)7J-s8^XenGy43e3YoYvEF3QyOXyW)#btDnYbFzwF>B(ZqaA+-VAU-DQ9%#~O1Q3T zUOvBw$)c*!5X!1gr!sKIV)!(b^-uZkkC8KW9YjtaovtAbqTec0O@>lo-JK}RM&{xB zOqD1+yRNts?NUc!?g)eIUQ65Bgg;5K#O`XDpWT=F33cg|{!3@^$PSL>I!*&^r+o&e z^PVcgDl`HZ?tKhd&&kNhad2@J_zPbCC&_iEUz87_;OIVExTL&%adGW10P)vGUA0eGH`DB2a?Q(9RWZJ*irWbz0yLwAS? zvU2iZRsz~)sJVQBW)WY#b#_WWqy|Cb;>I#Kjlt^$E%+|iBIC$OH=8a zl?$c@kKq;rMP)SV;^e1Z0@l~p5fKsbk+PLe%5jK<$VU)y^u!guEf16rmc9GTZa3l3 zzPxjaRiPCIOD<;bBqH@0HUA+4Vv9k3nikJugKJ{jj5fH{6$oaSkA3cNX>p z)R=`u(rYm0gC^PFCqFktJ(_rdWsFMqMQeHnXdZ9CKm!7i{Kjsl9-Evlua1-m_m4D% z<`HG>3^J6^{RKMs>6DgH^FE+C?YFSW%Zf+4rwEnxBSP|?cI&ufc@@HsRFe4!&yj_x z_ATfRxyhIw6AVL^iOn#bWHge?okn_|0R>OK*tb&|_%s|W*&vqm0i(yc)DdmcCMLCg zE#ttr+0TrXsVezb4{Nijjm&u>Zre7951C;qci&`-ryB9$*$$~QTFe#n3^D)Mbkl6N zG~Q4o1!>`=&#`iHb+5ENOG`ATYKv}&p`Uj;FIJjuY}%faLH$8hm0~Uc&sgzr)l}Q0 zRJMwYmSdP4zjZ5iM*_wmo&#(kuFHCvI0l+Q}0YqT6f zFc_@U^KcsK^^cbGlgwvf`>-qcZ!z*^C(6w&bs*jew|2&N@j%?VbpNlrNwKkiB4>%> zB7jHp3k#u~%QTnnQwO7ed>cKC*-V9$&8q8b{=M;*h zQH&PWK7GvG++6y{rk+b^6nJcilJS96^X_yFOWtxr88`}xn_-C?n5;11?mjoZo3F}TuI40w?|FoEdDr=RqEav-@IYex(<(}d)g}UWH~$t6Kyx~HdMPs6 zsB>Kjaja9CHcqVb-#=;B77rsp=Bo96&*>e|9G}SaL=4CKB{tpMcqLK_j*hWOidL&} z?(s?7r05LcUqSi!dAhDXkVgP-&qj4B%IRu?Dg}TzW(5A4hdm=mUfppH3;x%dJq}Ez zsZl)#2rc#X<|wJt(bmN|+^^0U(>_G2KU%M11c;=(Jc~fc9!}-ugb=!vt{f{5edb;B zE{H_y)#c^46Swf3qO!ORWa9SWhi0hx&y38>z65(%p2+={1l-U3XNh}gxC~&|bANxo zwY8O%D45bSOTY-jkP|Z=Hl1SSg)&zEb4C8UQ$V;6UcGt{H~*|!;gyb$k1zr?q=o^G zz$jxBL(LQUJ=DY2cJFbBck*_Hk2&UufI(5STeZru`30K8>@S(_A4M#cTj|C&H!MMw z4$bz<8A&H19%|S~24Um;c613?S~Up|*)r|eEww$Wp~At10}I>gy#+vJ)nyW(HZ`Ta z!<@bEF#CTQV^utJoHP1^SMo}G)_N$&3L0@xL|G6ZqHLj1%5*fRdsaW9jrD0Tj@;w= zVCrSw;J`mWC5#6n)y2hS!1c!FUq3H%8+h2(EMMxQaG;O=PsbT%6_n-CGAZqZ>mNf* z!$E_4e=^?@5-N{M+;{#LX-G*zqA2N#aG$Xg8T%tGc}V7P67if2K8wZMa4Tm;`Gov1 z76-z6zpP`V{iTS&(h*xuh8}TW<9ap<@I&=4-dAn&n5UE`DfSO?1l@gVTFG7U@oaPY^w?sf zSzzqS@95{mo+mzJ=IXdyys+@;E_cxHZ_CJ)SY?yj1F^~ew1hIfqi*{IIxN!IWH=#I zpYz?9LrABSN7>9_Jm+IJfe`zEC_xWZJqc14u<-Q0yZ@K3x55;;AXMv%5(SEddOU3O zBtwY2!wFO0e{|P7cLr-}!3T5m*O&o<;(PFFY^uGeshGI6+Z!#sb0f(JxTcQ8yzoaQ z@}D%f=zkNn|5iOS&%d$Dx4u_!i$&3mUZ4kqsCli))8`8i=&&vjVa|ZeN%z602TvoL z4+Yv6=1IkL-XG4z$r&K}9O)h)_^TCdZ9vmcSz5^l{IC7p1PKf|5)(-f9f^2nQS*Pt zilC5k_yd@~Z374dRvs9wnsk>=J6u0TCX>kqBfCMH$v4j@bxHLR&0}iG6gXRxf+9whqb|Ir#in<2M#UlanGE)vx8CPfw_KD%A?bW zu=uYOqN&@<;`Oujrdn5_zGaPe!F8BvfX@Xhu#qXP;GLfI@UGw*2e1c%T}`Al7jr6ZTI<^)zs9T1WeO`Qi8-M_<3W!8S=g=_(f;?c;r>NvSap4T~yNJ562o^`XRgjM9m zU%j6gEa!Ds9e3v{)bo$x5MLa_;1sc_m zM`cWC03xR(3nK2#gha(6MB*~v&=sHcMOrOjk#~VOzj`SA2H3x}*C#Si`(EPAQpfW< zkNgRSf;r4U`LWH`oFKxOl9|J|B-aqCXXDXyR*5ricW1I90TC~KIgyX$n^$8T5GGy4 zu;HMzAKi*;7H{JF0{as@)yEnv7>6Oww#mlL>o+d-0vm)LoI1K!SffzVD*gmJlqE8_0=3jWTNPCx4ajlpuSx2K&Ni+F}}0VbTtngoxqBO!|_qJSSQ+xJXyQ%m7@q6%>Bl{MRJ%N zyb-J-a|ni^7N+#A@Jd*RP38@|T z*w|QWW-|I3Z!fQK;|RM({I|A{`WrtU40dA#cjI)MSs+rix3{mgx$UgBd!WDigj*xO z!&eeRUst}=@|KQ17?RB(BI>D5QtzdpTMM-#VA%Sc9B2tta?Z%epb+{J#w25b?1dg= zj*i+(k_9;CMUX+L0T9;WDAB$H8#b;^O7luR9L-j3;=w2bWQp6@C+Yk{i;L+VT9IBTADQJ5G!z~UYYx9S_6}Jg zQwG729(M3VXsnp^{E@jrPhCvEs0Ur$2tX)8mmzd1-#9rVz-y+;qd z3N89biQ$Eu5GLYaO&3g#EbL~)npTPW4$MP_DU7dR!3S}(;0a?a_1VLxWGJ`9!ggb% z3p)woC9x7gPw95*LIxlQW7w-pk-6(?WLeV<>7U1HU8iC!5~aMu;Kc!N-#19{Wm{J* zW-?F!4_IcD@z6WuP(du^(&+p>G)fGj$nWCZ@4-=s1#mP{MA9`n?afJ~8~((~u4dEH zm+cZi(VRg0pzUH!_@Od<1np_%i|Wa_;8$!xNkTQ_zlZsggUi6I0p9g-tv^~4}imkueJdaRo?l$@t~3%;OQU$;_qA+<8n@4 zNpaHU+_qrw`Nv9>!r!28N=j%UI7f}AZwQt~vVU-}_iyU&w(G9o#qQ9}eB?6X1K4p_ zT~1zXvJ{cZ$|*}LM-YS$QIO*K)BSL~kORf9c&d=C^>|#A%ja#_&(V*8_4QMo9g2Qu zWGC$(H%|Kcl;r%Gd>E9zS5`_7(``h?V`E*#pQjK_cZyn6BU;4=S8 z&>-y-o1ny_FI=BlRU0_F*kRGg6Sg0WiwN-T=Dj<8hPvB2eiO~I>F@tl^PLGXwEJDxTzmVymxPDiWf+D&2}z=y+y%UQ z!gZ-c!4`YaP)W&&J=Tbw{hZ<*Q5Z60Bm~7MCr8lN7iDB57xJ90R>=YC=Qd0M5~JYM znzp;w=jR7^N~(MQ=|j3QSX{QcHq;{cvH3VD6^uqlK*mO=q*(Ev<1Iu$%sgNS zOmbv-e$KGqr*)c7JR1+eN?O_-=-B4lh*3rMLh{pS!(jxyJpn#EJg{;QEQkP+PZob= z3&5x)mY;T`yD}Fd=4x^a#4$R(u<{inA~GimyhH+kU6YdEs8Lb3?>5TSdYmCv^l^Z4 zz?>#XSMTiMUf0I<$K0&&@fudMbYghBh^IZj-B?SN4ksrF_hkKfXLsb$`*?E^GfDkd zAkEjH+qCF+mN-InCKnx$eE&rRZ28_XrNqi5&`y?=-+R$mZEYwJXrXu@Ai$&C7D$sT ziHjB5(tSAE?x9@8gPbK)XNdIhA0pDDS?6l2B@#T|@Rfm1Qao?FvO1?Qt;OMVb-tRt zWr9YEnjF8uavT_^UMNZ|!GcEvi%uNkpe)@9M%wD^tW|iuy5{Z zS)mGc@je*pJ4~`}Pd ze}-BOaC&>sd+OPJ6HOx~B`FvACLoRgI>rbMetGQdY%7{6S3CUj`TFKMj4^n9{Vyez zM1Fp=m38q)zl`*(+#ILouzpUoKpjyD5kA5@>H@o)%e`6DS}}uy^qZ1ORtZrfbB+4f zJylQjd;yGkfJTGU;WX#J`R~R*V#ttL|6S~C&(6YAQv8HmWvl7t3riEJgp+cU03I?F zBp=|G{eT~gg2d&&9LIs!1O{%NuP-&E+`S?a=Xa4M7J3Iy)YpRNw7c;^nwlSsEykJ3 z0)lnfBa2S)<0z?R^3O=|`jB1}AD`oHAGS-bIK8(D+Y>e92{G84=DE-AXrfK9WOr)6 zk8lvle3J%yACtt6z1+Nh=rw7GdG4k6F`e%2G{*@Z)7H7t_wW2HGxaXzS5nC zX|k25A-C{^)T5`fx+AK)jF!C`Ul#%<%3kaHQ6O1_r1zF8CYzCLZoxF=|3Aa0l ApplyConfiguration(JToken configJson) + { + configData.LoadValuesFromJson(configJson); + var pairs = new Dictionary { + { "username", configData.Username.Value }, + { "password", configData.Password.Value }, + { "form", "login" }, + { "rememberme[]", "1" } + }; + + var loginPage = await RequestStringWithCookiesAndRetry(LoginUrl, null, null); + + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SearchUrl, SiteLink); + await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () => + { + CQ dom = result.Content; + var errorMessage = dom[".ui-state-error"].Text().Trim(); + throw new ExceptionWithConfigData(errorMessage, configData); + }); + + return IndexerConfigurationStatus.RequiresTesting; + } + + public async Task> PerformQuery(TorznabQuery query) + { + var releases = new List(); + var searchString = query.GetQueryString(); + var searchUrl = SearchUrl; + var queryCollection = new NameValueCollection(); + + queryCollection.Add("total", "146"); // Not sure what this is about but its required! + + var cat = "0"; + var queryCats = MapTorznabCapsToTrackers(query); + if (queryCats.Count == 1) + { + cat = queryCats.First().ToString(); + } + + queryCollection.Add("cat", cat); + queryCollection.Add("searchin", "filename"); + queryCollection.Add("search", searchString); + queryCollection.Add("page", "1"); + searchUrl += "?" + queryCollection.GetQueryString(); + + var extraHeaders = new Dictionary() + { + { "X-Requested-With", "XMLHttpRequest" } + }; + + var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrlReferer, extraHeaders); + + var results = response.Content; + try + { + CQ dom = results; + + var rows = dom["tr"]; + foreach (var row in rows.Skip(1)) + { + var release = new ReleaseInfo(); + var qRow = row.Cq(); + var qTitleLink = qRow.Find("td:eq(1) a:eq(0)").First(); + release.Title = qTitleLink.Find("strong").Text().Trim(); + + // If we search an get no results, we still get a table just with no info. + if (string.IsNullOrWhiteSpace(release.Title)) + { + break; + } + + release.Description = release.Title; + release.Guid = new Uri(qTitleLink.Attr("href")); + release.Comments = release.Guid; + + var dateString = qRow.Find("td:eq(4)").Text(); + release.PublishDate = DateTime.ParseExact(dateString, "dd MMM yy", CultureInfo.InvariantCulture); + + var qLink = qRow.Find("td:eq(2) a"); + release.Link = new Uri(qLink.Attr("href")); + + var sizeStr = qRow.Find("td:eq(5)").Text(); + release.Size = ReleaseInfo.GetBytes(sizeStr); + + var connections = qRow.Find("td:eq(7)").Text().Trim().Split("/".ToCharArray(),StringSplitOptions.RemoveEmptyEntries); + + release.Seeders = ParseUtil.CoerceInt(connections[0].Trim()); + release.Peers = ParseUtil.CoerceInt(connections[1].Trim()) + release.Seeders; + + var rCat = row.Cq().Find("td:eq(0) a").First().Attr("href"); + var rCatIdx = rCat.IndexOf("cat="); + if (rCatIdx > -1) + { + rCat = rCat.Substring(rCatIdx + 4); + } + + release.Category = MapTrackerCatToNewznab(rCat); + + releases.Add(release); + } + } + catch (Exception ex) + { + OnParseError(results, ex); + } + + return releases; + } + } +} diff --git a/src/Jackett/Indexers/BaseIndexer.cs b/src/Jackett/Indexers/BaseIndexer.cs index 71cf91979..2b7efef21 100644 --- a/src/Jackett/Indexers/BaseIndexer.cs +++ b/src/Jackett/Indexers/BaseIndexer.cs @@ -1,451 +1,452 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Jackett.Models; -using Newtonsoft.Json.Linq; -using NLog; -using Jackett.Services; -using Jackett.Utils; -using Jackett.Utils.Clients; -using AutoMapper; -using System.Threading; -using Jackett.Models.IndexerConfig; - -namespace Jackett.Indexers -{ - public abstract class BaseIndexer - { - public string SiteLink { get; private set; } - public string DisplayDescription { get; private set; } - public string DisplayName { get; private set; } - public string ID { get { return GetIndexerID(GetType()); } } - - public bool IsConfigured { get; protected set; } - public TorznabCapabilities TorznabCaps { get; private set; } - protected Logger logger; - protected IIndexerManagerService indexerService; - protected static List cache = new List(); - protected static readonly TimeSpan cacheTime = new TimeSpan(0, 9, 0); - protected IWebClient webclient; - protected IProtectionService protectionService; - protected readonly string downloadUrlBase = ""; - - protected string CookieHeader - { - get { return configData.CookieHeader.Value; } - set { configData.CookieHeader.Value = value; } - } - - - - protected ConfigurationData configData; - - private List categoryMapping = new List(); - - public BaseIndexer(string name, string link, string description, IIndexerManagerService manager, IWebClient client, Logger logger, ConfigurationData configData, IProtectionService p, TorznabCapabilities caps = null, string downloadBase = null) - { - if (!link.EndsWith("/")) - throw new Exception("Site link must end with a slash."); - - DisplayName = name; - DisplayDescription = description; - SiteLink = link; - this.logger = logger; - indexerService = manager; - webclient = client; - protectionService = p; - this.downloadUrlBase = downloadBase; - - this.configData = configData; - - if (caps == null) - caps = TorznabUtil.CreateDefaultTorznabTVCaps(); - TorznabCaps = caps; - - } - - public IEnumerable CleanLinks(IEnumerable releases) - { - if (string.IsNullOrEmpty(downloadUrlBase)) - return releases; - foreach (var release in releases) - { - if (release.Link.ToString().StartsWith(downloadUrlBase)) - { - release.Link = new Uri(release.Link.ToString().Substring(downloadUrlBase.Length), UriKind.Relative); - } - } - - return releases; - } - - public Uri UncleanLink(Uri link) - { - return new Uri(downloadUrlBase + link.ToString(), UriKind.RelativeOrAbsolute); - } - - protected int MapTrackerCatToNewznab(string input) - { - if (null != input) - { - input = input.ToLowerInvariant(); - var mapping = categoryMapping.Where(m => m.TrackerCategory.ToLowerInvariant() == input.ToLowerInvariant()).FirstOrDefault(); - if (mapping != null) - { - return mapping.NewzNabCategory; - } - } - return 0; - } - - public static string GetIndexerID(Type type) - { - return StringUtil.StripNonAlphaNumeric(type.Name.ToLowerInvariant()); - } - - public virtual Task GetConfigurationForSetup() - { - return Task.FromResult(configData); - } - - public virtual void ResetBaseConfig() - { - CookieHeader = string.Empty; - IsConfigured = false; - } - - protected virtual void SaveConfig() - { - indexerService.SaveConfig(this as IIndexer, configData.ToJson(protectionService, forDisplay: false)); - } - - protected void OnParseError(string results, Exception ex) - { - var fileName = string.Format("Error on {0} for {1}.txt", DateTime.Now.ToString("yyyyMMddHHmmss"), DisplayName); - var spacing = string.Join("", Enumerable.Repeat(Environment.NewLine, 5)); - var fileContents = string.Format("{0}{1}{2}", ex, spacing, results); - logger.Error(fileName + fileContents); - } - - protected void CleanCache() - { - foreach (var expired in cache.Where(i => i.Created - DateTime.Now > cacheTime).ToList()) - { - cache.Remove(expired); - } - } - - protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) - { - var byteResult = new WebClientByteResult(); - // Map to byte - Mapper.Map(response, byteResult); - await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies); - // Map to string - Mapper.Map(byteResult, response); - } - - protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) - { - // Follow up to 5 redirects - for (int i = 0; i < 5; i++) - { - if (!response.IsRedirect) - break; - await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies); - } - } - - private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) - { - if (incomingResponse.IsRedirect) - { - // Do redirect - var redirectedResponse = await webclient.GetBytes(new WebRequest() - { - Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo, - Referer = referrer, - Cookies = overrideCookies ?? CookieHeader - }); - Mapper.Map(redirectedResponse, incomingResponse); - } - } - - - protected void LoadLegacyCookieConfig(JToken jsonConfig) - { - string legacyCookieHeader = (string)jsonConfig["cookie_header"]; - if (!string.IsNullOrEmpty(legacyCookieHeader)) - { - CookieHeader = legacyCookieHeader; - } - else - { - // Legacy cookie key - var jcookies = jsonConfig["cookies"]; - if (jcookies is JArray) - { - var array = (JArray)jcookies; - legacyCookieHeader = string.Empty; - for (int i = 0; i < array.Count; i++) - { - if (i != 0) - legacyCookieHeader += "; "; - legacyCookieHeader += array[i]; - } - CookieHeader = legacyCookieHeader; - } - else if (jcookies != null) - { - CookieHeader = (string)jcookies; - } - } - } - - public virtual void LoadFromSavedConfiguration(JToken jsonConfig) - { - if (jsonConfig is JArray) - { - configData.LoadValuesFromJson(jsonConfig, protectionService); - IsConfigured = true; - } - // read and upgrade old settings file format - else if (jsonConfig is Object) - { - LoadLegacyCookieConfig(jsonConfig); - SaveConfig(); - IsConfigured = true; - } - } - - public async virtual Task Download(Uri link) - { - var response = await RequestBytesWithCookiesAndRetry(link.ToString()); - return response.Content; - } - - protected async Task RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null) - { - Exception lastException = null; - for (int i = 0; i < 3; i++) - { - try - { - return await RequestBytesWithCookies(url, cookieOverride); - } - catch (Exception e) - { - logger.Error(string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message)); - lastException = e; - } - await Task.Delay(500); - } - - throw lastException; - } - - protected async Task RequestStringWithCookies(string url, string cookieOverride = null, string referer = null) - { - var request = new Utils.Clients.WebRequest() - { - Url = url, - Type = RequestType.GET, - Cookies = CookieHeader, - Referer = referer - }; - - if (cookieOverride != null) - request.Cookies = cookieOverride; - return await webclient.GetString(request); - } - - protected async Task RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null) - { - Exception lastException = null; - for (int i = 0; i < 3; i++) - { - try - { - return await RequestStringWithCookies(url, cookieOverride, referer); - } - catch (Exception e) - { - logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); - lastException = e; - } - await Task.Delay(500); - } - - throw lastException; - } - - protected async Task RequestBytesWithCookies(string url, string cookieOverride = null) - { - var request = new Utils.Clients.WebRequest() - { - Url = url, - Type = RequestType.GET, - Cookies = cookieOverride ?? CookieHeader - }; - - if (cookieOverride != null) - request.Cookies = cookieOverride; - return await webclient.GetBytes(request); - } - - protected async Task PostDataWithCookies(string url, IEnumerable> data, string cookieOverride = null) - { - var request = new Utils.Clients.WebRequest() - { - Url = url, - Type = RequestType.POST, - Cookies = cookieOverride ?? CookieHeader, - PostData = data - }; - return await webclient.GetString(request); - } - - protected async Task PostDataWithCookiesAndRetry(string url, IEnumerable> data, string cookieOverride = null) - { - Exception lastException = null; - for (int i = 0; i < 3; i++) - { - try - { - return await PostDataWithCookies(url, data, cookieOverride); - } - catch (Exception e) - { - logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); - lastException = e; - } - await Task.Delay(500); - } - - throw lastException; - } - - protected async Task RequestLoginAndFollowRedirect(string url, IEnumerable> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null) - { - var request = new Utils.Clients.WebRequest() - { - Url = url, - Type = RequestType.POST, - Cookies = cookies, - Referer = referer, - PostData = data - }; - var response = await webclient.GetString(request); - var firstCallCookies = response.Cookies; - - if (response.IsRedirect) - { - await FollowIfRedirect(response, request.Url, null, response.Cookies); - } - - if (returnCookiesFromFirstCall) - { - response.Cookies = firstCallCookies; - } - - return response; - } - - protected async Task ConfigureIfOK(string cookies, bool isLoggedin, Func onError) - { - if (isLoggedin) - { - CookieHeader = cookies; - SaveConfig(); - IsConfigured = true; - } - else - { - await onError(); - } - } - - public virtual IEnumerable FilterResults(TorznabQuery query, IEnumerable results) - { - foreach (var result in results) - { - if (query.Categories.Length == 0 || query.Categories.Contains(result.Category) || result.Category == 0 || TorznabCatType.QueryContainsParentCategory(query.Categories, result.Category)) - { - yield return result; - } - } - } - - protected void AddCategoryMapping(string trackerCategory, int newznabCategory) - { - categoryMapping.Add(new CategoryMapping(trackerCategory, newznabCategory)); - } - - protected void AddCategoryMapping(int trackerCategory, TorznabCategory newznabCategory) - { - categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory.ID)); - if (!TorznabCaps.Categories.Contains(newznabCategory)) - TorznabCaps.Categories.Add(newznabCategory); - } - - protected void AddCategoryMapping(string trackerCategory, TorznabCategory newznabCategory) - { - categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory.ID)); - if (!TorznabCaps.Categories.Contains(newznabCategory)) - TorznabCaps.Categories.Add(newznabCategory); - } - - protected void AddCategoryMapping(int trackerCategory, int newznabCategory) - { - categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory)); - } - - protected void AddMultiCategoryMapping(TorznabCategory newznabCategory, params int[] trackerCategories) - { - foreach (var trackerCat in trackerCategories) - { - categoryMapping.Add(new CategoryMapping(trackerCat.ToString(), newznabCategory.ID)); - } - } - - protected void AddMultiCategoryMapping(int trackerCategory, params TorznabCategory[] newznabCategories) - { - foreach (var newznabCat in newznabCategories) - { - categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCat.ID)); - } - } - - protected List MapTorznabCapsToTrackers(TorznabQuery query, bool mapChildrenCatsToParent = false) - { - var result = new List(); - foreach (var cat in query.Categories) - { - var queryCats = new List { cat }; - var newznabCat = TorznabCatType.AllCats.FirstOrDefault(c => c.ID == cat); - if (newznabCat != null) - { - queryCats.AddRange(newznabCat.SubCategories.Select(c => c.ID)); - } - - if (mapChildrenCatsToParent) - { - var parentNewznabCat = TorznabCatType.AllCats.FirstOrDefault(c => c.SubCategories.Contains(newznabCat)); - if (parentNewznabCat != null) - { - queryCats.Add(parentNewznabCat.ID); - } - } - - foreach (var mapping in categoryMapping.Where(c => queryCats.Contains(c.NewzNabCategory))) - { - result.Add(mapping.TrackerCategory); - } - } - - return result.Distinct().ToList(); - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Jackett.Models; +using Newtonsoft.Json.Linq; +using NLog; +using Jackett.Services; +using Jackett.Utils; +using Jackett.Utils.Clients; +using AutoMapper; +using System.Threading; +using Jackett.Models.IndexerConfig; + +namespace Jackett.Indexers +{ + public abstract class BaseIndexer + { + public string SiteLink { get; private set; } + public string DisplayDescription { get; private set; } + public string DisplayName { get; private set; } + public string ID { get { return GetIndexerID(GetType()); } } + + public bool IsConfigured { get; protected set; } + public TorznabCapabilities TorznabCaps { get; private set; } + protected Logger logger; + protected IIndexerManagerService indexerService; + protected static List cache = new List(); + protected static readonly TimeSpan cacheTime = new TimeSpan(0, 9, 0); + protected IWebClient webclient; + protected IProtectionService protectionService; + protected readonly string downloadUrlBase = ""; + + protected string CookieHeader + { + get { return configData.CookieHeader.Value; } + set { configData.CookieHeader.Value = value; } + } + + + + protected ConfigurationData configData; + + private List categoryMapping = new List(); + + public BaseIndexer(string name, string link, string description, IIndexerManagerService manager, IWebClient client, Logger logger, ConfigurationData configData, IProtectionService p, TorznabCapabilities caps = null, string downloadBase = null) + { + if (!link.EndsWith("/")) + throw new Exception("Site link must end with a slash."); + + DisplayName = name; + DisplayDescription = description; + SiteLink = link; + this.logger = logger; + indexerService = manager; + webclient = client; + protectionService = p; + this.downloadUrlBase = downloadBase; + + this.configData = configData; + + if (caps == null) + caps = TorznabUtil.CreateDefaultTorznabTVCaps(); + TorznabCaps = caps; + + } + + public IEnumerable CleanLinks(IEnumerable releases) + { + if (string.IsNullOrEmpty(downloadUrlBase)) + return releases; + foreach (var release in releases) + { + if (release.Link.ToString().StartsWith(downloadUrlBase)) + { + release.Link = new Uri(release.Link.ToString().Substring(downloadUrlBase.Length), UriKind.Relative); + } + } + + return releases; + } + + public Uri UncleanLink(Uri link) + { + return new Uri(downloadUrlBase + link.ToString(), UriKind.RelativeOrAbsolute); + } + + protected int MapTrackerCatToNewznab(string input) + { + if (null != input) + { + input = input.ToLowerInvariant(); + var mapping = categoryMapping.Where(m => m.TrackerCategory.ToLowerInvariant() == input.ToLowerInvariant()).FirstOrDefault(); + if (mapping != null) + { + return mapping.NewzNabCategory; + } + } + return 0; + } + + public static string GetIndexerID(Type type) + { + return StringUtil.StripNonAlphaNumeric(type.Name.ToLowerInvariant()); + } + + public virtual Task GetConfigurationForSetup() + { + return Task.FromResult(configData); + } + + public virtual void ResetBaseConfig() + { + CookieHeader = string.Empty; + IsConfigured = false; + } + + protected virtual void SaveConfig() + { + indexerService.SaveConfig(this as IIndexer, configData.ToJson(protectionService, forDisplay: false)); + } + + protected void OnParseError(string results, Exception ex) + { + var fileName = string.Format("Error on {0} for {1}.txt", DateTime.Now.ToString("yyyyMMddHHmmss"), DisplayName); + var spacing = string.Join("", Enumerable.Repeat(Environment.NewLine, 5)); + var fileContents = string.Format("{0}{1}{2}", ex, spacing, results); + logger.Error(fileName + fileContents); + } + + protected void CleanCache() + { + foreach (var expired in cache.Where(i => i.Created - DateTime.Now > cacheTime).ToList()) + { + cache.Remove(expired); + } + } + + protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) + { + var byteResult = new WebClientByteResult(); + // Map to byte + Mapper.Map(response, byteResult); + await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies); + // Map to string + Mapper.Map(byteResult, response); + } + + protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) + { + // Follow up to 5 redirects + for (int i = 0; i < 5; i++) + { + if (!response.IsRedirect) + break; + await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies); + } + } + + private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) + { + if (incomingResponse.IsRedirect) + { + // Do redirect + var redirectedResponse = await webclient.GetBytes(new WebRequest() + { + Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo, + Referer = referrer, + Cookies = overrideCookies ?? CookieHeader + }); + Mapper.Map(redirectedResponse, incomingResponse); + } + } + + + protected void LoadLegacyCookieConfig(JToken jsonConfig) + { + string legacyCookieHeader = (string)jsonConfig["cookie_header"]; + if (!string.IsNullOrEmpty(legacyCookieHeader)) + { + CookieHeader = legacyCookieHeader; + } + else + { + // Legacy cookie key + var jcookies = jsonConfig["cookies"]; + if (jcookies is JArray) + { + var array = (JArray)jcookies; + legacyCookieHeader = string.Empty; + for (int i = 0; i < array.Count; i++) + { + if (i != 0) + legacyCookieHeader += "; "; + legacyCookieHeader += array[i]; + } + CookieHeader = legacyCookieHeader; + } + else if (jcookies != null) + { + CookieHeader = (string)jcookies; + } + } + } + + public virtual void LoadFromSavedConfiguration(JToken jsonConfig) + { + if (jsonConfig is JArray) + { + configData.LoadValuesFromJson(jsonConfig, protectionService); + IsConfigured = true; + } + // read and upgrade old settings file format + else if (jsonConfig is Object) + { + LoadLegacyCookieConfig(jsonConfig); + SaveConfig(); + IsConfigured = true; + } + } + + public async virtual Task Download(Uri link) + { + var response = await RequestBytesWithCookiesAndRetry(link.ToString()); + return response.Content; + } + + protected async Task RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null) + { + Exception lastException = null; + for (int i = 0; i < 3; i++) + { + try + { + return await RequestBytesWithCookies(url, cookieOverride); + } + catch (Exception e) + { + logger.Error(string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message)); + lastException = e; + } + await Task.Delay(500); + } + + throw lastException; + } + + protected async Task RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary headers = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.GET, + Cookies = CookieHeader, + Referer = referer, + Headers = headers + }; + + if (cookieOverride != null) + request.Cookies = cookieOverride; + return await webclient.GetString(request); + } + + protected async Task RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary headers = null) + { + Exception lastException = null; + for (int i = 0; i < 3; i++) + { + try + { + return await RequestStringWithCookies(url, cookieOverride, referer, headers); + } + catch (Exception e) + { + logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); + lastException = e; + } + await Task.Delay(500); + } + + throw lastException; + } + + protected async Task RequestBytesWithCookies(string url, string cookieOverride = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.GET, + Cookies = cookieOverride ?? CookieHeader + }; + + if (cookieOverride != null) + request.Cookies = cookieOverride; + return await webclient.GetBytes(request); + } + + protected async Task PostDataWithCookies(string url, IEnumerable> data, string cookieOverride = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.POST, + Cookies = cookieOverride ?? CookieHeader, + PostData = data + }; + return await webclient.GetString(request); + } + + protected async Task PostDataWithCookiesAndRetry(string url, IEnumerable> data, string cookieOverride = null) + { + Exception lastException = null; + for (int i = 0; i < 3; i++) + { + try + { + return await PostDataWithCookies(url, data, cookieOverride); + } + catch (Exception e) + { + logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); + lastException = e; + } + await Task.Delay(500); + } + + throw lastException; + } + + protected async Task RequestLoginAndFollowRedirect(string url, IEnumerable> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.POST, + Cookies = cookies, + Referer = referer, + PostData = data + }; + var response = await webclient.GetString(request); + var firstCallCookies = response.Cookies; + + if (response.IsRedirect) + { + await FollowIfRedirect(response, request.Url, null, response.Cookies); + } + + if (returnCookiesFromFirstCall) + { + response.Cookies = firstCallCookies; + } + + return response; + } + + protected async Task ConfigureIfOK(string cookies, bool isLoggedin, Func onError) + { + if (isLoggedin) + { + CookieHeader = cookies; + SaveConfig(); + IsConfigured = true; + } + else + { + await onError(); + } + } + + public virtual IEnumerable FilterResults(TorznabQuery query, IEnumerable results) + { + foreach (var result in results) + { + if (query.Categories.Length == 0 || query.Categories.Contains(result.Category) || result.Category == 0 || TorznabCatType.QueryContainsParentCategory(query.Categories, result.Category)) + { + yield return result; + } + } + } + + protected void AddCategoryMapping(string trackerCategory, int newznabCategory) + { + categoryMapping.Add(new CategoryMapping(trackerCategory, newznabCategory)); + } + + protected void AddCategoryMapping(int trackerCategory, TorznabCategory newznabCategory) + { + categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory.ID)); + if (!TorznabCaps.Categories.Contains(newznabCategory)) + TorznabCaps.Categories.Add(newznabCategory); + } + + protected void AddCategoryMapping(string trackerCategory, TorznabCategory newznabCategory) + { + categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory.ID)); + if (!TorznabCaps.Categories.Contains(newznabCategory)) + TorznabCaps.Categories.Add(newznabCategory); + } + + protected void AddCategoryMapping(int trackerCategory, int newznabCategory) + { + categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory)); + } + + protected void AddMultiCategoryMapping(TorznabCategory newznabCategory, params int[] trackerCategories) + { + foreach (var trackerCat in trackerCategories) + { + categoryMapping.Add(new CategoryMapping(trackerCat.ToString(), newznabCategory.ID)); + } + } + + protected void AddMultiCategoryMapping(int trackerCategory, params TorznabCategory[] newznabCategories) + { + foreach (var newznabCat in newznabCategories) + { + categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCat.ID)); + } + } + + protected List MapTorznabCapsToTrackers(TorznabQuery query, bool mapChildrenCatsToParent = false) + { + var result = new List(); + foreach (var cat in query.Categories) + { + var queryCats = new List { cat }; + var newznabCat = TorznabCatType.AllCats.FirstOrDefault(c => c.ID == cat); + if (newznabCat != null) + { + queryCats.AddRange(newznabCat.SubCategories.Select(c => c.ID)); + } + + if (mapChildrenCatsToParent) + { + var parentNewznabCat = TorznabCatType.AllCats.FirstOrDefault(c => c.SubCategories.Contains(newznabCat)); + if (parentNewznabCat != null) + { + queryCats.Add(parentNewznabCat.ID); + } + } + + foreach (var mapping in categoryMapping.Where(c => queryCats.Contains(c.NewzNabCategory))) + { + result.Add(mapping.TrackerCategory); + } + } + + return result.Distinct().ToList(); + } + } +} diff --git a/src/Jackett/Jackett.csproj b/src/Jackett/Jackett.csproj index b84a20994..71a193b74 100644 --- a/src/Jackett/Jackett.csproj +++ b/src/Jackett/Jackett.csproj @@ -1,609 +1,613 @@ - - - - - Debug - AnyCPU - {E636D5F8-68B4-4903-B4ED-CCFD9C9E899F} - Library - Properties - Jackett - Jackett - v4.5 - 512 - - publish\ - true - Disk - false - Foreground - 7 - Days - false - false - true - 0 - 1.0.0.%2a - false - false - true - - - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - - - - - - - - - ..\packages\Autofac.3.5.2\lib\net40\Autofac.dll - True - - - ..\packages\Autofac.Owin.3.1.0\lib\net45\Autofac.Integration.Owin.dll - True - - - ..\packages\Autofac.WebApi2.3.4.0\lib\net45\Autofac.Integration.WebApi.dll - True - - - ..\packages\Autofac.WebApi2.Owin.3.2.0\lib\net45\Autofac.Integration.WebApi.Owin.dll - True - - - ..\packages\AutoMapper.4.0.4\lib\net45\AutoMapper.dll - True - - - ..\packages\CsQuery.1.3.4\lib\net40\CsQuery.dll - True - - - ..\packages\Microsoft.AspNet.Identity.Core.2.2.1\lib\net45\Microsoft.AspNet.Identity.Core.dll - True - - - ..\packages\Microsoft.Owin.3.0.1\lib\net45\Microsoft.Owin.dll - True - - - ..\packages\Microsoft.Owin.FileSystems.3.0.1\lib\net45\Microsoft.Owin.FileSystems.dll - True - - - ..\packages\Microsoft.Owin.Host.HttpListener.3.0.1\lib\net45\Microsoft.Owin.Host.HttpListener.dll - True - - - ..\packages\Microsoft.Owin.Host.SystemWeb.3.0.1\lib\net45\Microsoft.Owin.Host.SystemWeb.dll - True - - - ..\packages\Microsoft.Owin.Hosting.3.0.1\lib\net45\Microsoft.Owin.Hosting.dll - True - - - ..\packages\Microsoft.Owin.StaticFiles.3.0.1\lib\net45\Microsoft.Owin.StaticFiles.dll - True - - - ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll - True - - - ..\packages\NLog.4.0.1\lib\net45\NLog.dll - True - - - ..\packages\NLog.Windows.Forms.2.0.0.0\lib\net35\NLog.Windows.Forms.dll - True - - - ..\packages\Owin.1.0\lib\net40\Owin.dll - True - - - - - - - - ..\packages\Microsoft.Net.Http.2.2.29\lib\net45\System.Net.Http.Extensions.dll - True - - - ..\packages\Microsoft.AspNet.WebApi.Client.5.2.3\lib\net45\System.Net.Http.Formatting.dll - True - - - ..\packages\Microsoft.Net.Http.2.2.29\lib\net45\System.Net.Http.Primitives.dll - True - - - - - - - ..\packages\Microsoft.AspNet.WebApi.Core.5.2.3\lib\net45\System.Web.Http.dll - True - - - ..\packages\Microsoft.AspNet.WebApi.Owin.5.2.3\lib\net45\System.Web.Http.Owin.dll - True - - - ..\packages\Microsoft.AspNet.WebApi.Tracing.5.2.3\lib\net45\System.Web.Http.Tracing.dll - True - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - True - True - TorznabCatType.tt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - True - True - Resources.resx - - - - - - - - - - - - - - - - - - - - - - Designer - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - - - Always - - - Designer - - - - - - ResXFileCodeGenerator - Resources.Designer.cs - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - TextTemplatingFileGenerator - TorznabCatType.generated.cs - - - - PreserveNewest - - - - - {74420A79-CC16-442C-8B1E-7C1B913844F0} - CurlSharp - - - - - False - Microsoft .NET Framework 4.5.1 %28x86 and x64%29 - true - - - False - .NET Framework 3.5 SP1 Client Profile - false - - - False - .NET Framework 3.5 SP1 - false - - - - - - + + + + + Debug + AnyCPU + {E636D5F8-68B4-4903-B4ED-CCFD9C9E899F} + Library + Properties + Jackett + Jackett + v4.5 + 512 + + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true + + + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + ..\packages\Autofac.3.5.2\lib\net40\Autofac.dll + True + + + ..\packages\Autofac.Owin.3.1.0\lib\net45\Autofac.Integration.Owin.dll + True + + + ..\packages\Autofac.WebApi2.3.4.0\lib\net45\Autofac.Integration.WebApi.dll + True + + + ..\packages\Autofac.WebApi2.Owin.3.2.0\lib\net45\Autofac.Integration.WebApi.Owin.dll + True + + + ..\packages\AutoMapper.4.0.4\lib\net45\AutoMapper.dll + True + + + ..\packages\CsQuery.1.3.4\lib\net40\CsQuery.dll + True + + + ..\packages\Microsoft.AspNet.Identity.Core.2.2.1\lib\net45\Microsoft.AspNet.Identity.Core.dll + True + + + ..\packages\Microsoft.Owin.3.0.1\lib\net45\Microsoft.Owin.dll + True + + + ..\packages\Microsoft.Owin.FileSystems.3.0.1\lib\net45\Microsoft.Owin.FileSystems.dll + True + + + ..\packages\Microsoft.Owin.Host.HttpListener.3.0.1\lib\net45\Microsoft.Owin.Host.HttpListener.dll + True + + + ..\packages\Microsoft.Owin.Host.SystemWeb.3.0.1\lib\net45\Microsoft.Owin.Host.SystemWeb.dll + True + + + ..\packages\Microsoft.Owin.Hosting.3.0.1\lib\net45\Microsoft.Owin.Hosting.dll + True + + + ..\packages\Microsoft.Owin.StaticFiles.3.0.1\lib\net45\Microsoft.Owin.StaticFiles.dll + True + + + ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll + True + + + ..\packages\NLog.4.0.1\lib\net45\NLog.dll + True + + + ..\packages\NLog.Windows.Forms.2.0.0.0\lib\net35\NLog.Windows.Forms.dll + True + + + ..\packages\Owin.1.0\lib\net40\Owin.dll + True + + + + + + + + ..\packages\Microsoft.Net.Http.2.2.29\lib\net45\System.Net.Http.Extensions.dll + True + + + ..\packages\Microsoft.AspNet.WebApi.Client.5.2.3\lib\net45\System.Net.Http.Formatting.dll + True + + + ..\packages\Microsoft.Net.Http.2.2.29\lib\net45\System.Net.Http.Primitives.dll + True + + + + + + + ..\packages\Microsoft.AspNet.WebApi.Core.5.2.3\lib\net45\System.Web.Http.dll + True + + + ..\packages\Microsoft.AspNet.WebApi.Owin.5.2.3\lib\net45\System.Web.Http.Owin.dll + True + + + ..\packages\Microsoft.AspNet.WebApi.Tracing.5.2.3\lib\net45\System.Web.Http.Tracing.dll + True + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + True + True + TorznabCatType.tt + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + True + True + Resources.resx + + + + + + + + + + + + + + + + + + + + + + Designer + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + Always + + + Designer + + + + + + ResXFileCodeGenerator + Resources.Designer.cs + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + TextTemplatingFileGenerator + TorznabCatType.generated.cs + + + + PreserveNewest + + + + + {74420A79-CC16-442C-8B1E-7C1B913844F0} + CurlSharp + + + + + False + Microsoft .NET Framework 4.5.1 %28x86 and x64%29 + true + + + False + .NET Framework 3.5 SP1 Client Profile + false + + + False + .NET Framework 3.5 SP1 + false + + + + + + - - - - - - - - - - - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - + --> + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + \ No newline at end of file diff --git a/src/Jackett/JackettModule.cs b/src/Jackett/JackettModule.cs index cccbbe192..988b61238 100644 --- a/src/Jackett/JackettModule.cs +++ b/src/Jackett/JackettModule.cs @@ -1,81 +1,82 @@ -using Autofac; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Autofac.Integration.WebApi; -using Jackett.Indexers; -using Jackett.Utils; -using Jackett.Utils.Clients; -using AutoMapper; -using Jackett.Models; - -namespace Jackett -{ - public class JackettModule : Module - { - protected override void Load(ContainerBuilder builder) - { - // Just register everything! - var thisAssembly = typeof(JackettModule).Assembly; - builder.RegisterAssemblyTypes(thisAssembly).Except().AsImplementedInterfaces().SingleInstance(); - builder.RegisterApiControllers(thisAssembly).InstancePerRequest(); - - // Register the best web client for the platform or the override - switch (Startup.ClientOverride) - { - case "httpclient": - builder.RegisterType().As(); - break; - case "safecurl": - builder.RegisterType().As(); - break; - case "libcurl": - builder.RegisterType().As(); - break; - case "automatic": - default: - if (System.Environment.OSVersion.Platform == PlatformID.Unix) - { - builder.RegisterType().As(); - } - else - { - builder.RegisterType().As(); - } - break; - } - - // Register indexers - foreach (var indexer in thisAssembly.GetTypes() - .Where(p => typeof(IIndexer).IsAssignableFrom(p) && !p.IsInterface) - .ToArray()) - { - builder.RegisterType(indexer).Named(BaseIndexer.GetIndexerID(indexer)); - } - - Mapper.CreateMap().ForMember(x => x.Content, opt => opt.Ignore()).AfterMap((be, str) => - { - str.Content = Encoding.UTF8.GetString(be.Content); - }); - - Mapper.CreateMap().ForMember(x => x.Content, opt => opt.Ignore()).AfterMap((str, be) => - { - if (!string.IsNullOrEmpty(str.Content)) - { - be.Content = Encoding.UTF8.GetBytes(str.Content); - } - }); - - Mapper.CreateMap(); - Mapper.CreateMap(); - Mapper.CreateMap(); - - Mapper.CreateMap().AfterMap((r, t) => - { - t.CategoryDesc = TorznabCatType.GetCatDesc(r.Category); - }); - } - } -} +using Autofac; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Autofac.Integration.WebApi; +using Jackett.Indexers; +using Jackett.Utils; +using Jackett.Utils.Clients; +using AutoMapper; +using Jackett.Models; + +namespace Jackett +{ + public class JackettModule : Module + { + protected override void Load(ContainerBuilder builder) + { + // Just register everything! + var thisAssembly = typeof(JackettModule).Assembly; + builder.RegisterAssemblyTypes(thisAssembly).Except().AsImplementedInterfaces().SingleInstance(); + builder.RegisterApiControllers(thisAssembly).InstancePerRequest(); + builder.RegisterType(); + + // Register the best web client for the platform or the override + switch (Startup.ClientOverride) + { + case "httpclient": + builder.RegisterType().As(); + break; + case "safecurl": + builder.RegisterType().As(); + break; + case "libcurl": + builder.RegisterType().As(); + break; + case "automatic": + default: + if (System.Environment.OSVersion.Platform == PlatformID.Unix) + { + builder.RegisterType().As(); + } + else + { + builder.RegisterType().As(); + } + break; + } + + // Register indexers + foreach (var indexer in thisAssembly.GetTypes() + .Where(p => typeof(IIndexer).IsAssignableFrom(p) && !p.IsInterface) + .ToArray()) + { + builder.RegisterType(indexer).Named(BaseIndexer.GetIndexerID(indexer)); + } + + Mapper.CreateMap().ForMember(x => x.Content, opt => opt.Ignore()).AfterMap((be, str) => + { + str.Content = Encoding.UTF8.GetString(be.Content); + }); + + Mapper.CreateMap().ForMember(x => x.Content, opt => opt.Ignore()).AfterMap((str, be) => + { + if (!string.IsNullOrEmpty(str.Content)) + { + be.Content = Encoding.UTF8.GetBytes(str.Content); + } + }); + + Mapper.CreateMap(); + Mapper.CreateMap(); + Mapper.CreateMap(); + + Mapper.CreateMap().AfterMap((r, t) => + { + t.CategoryDesc = TorznabCatType.GetCatDesc(r.Category); + }); + } + } +} diff --git a/src/Jackett/Utils/Clients/HttpWebClient.cs b/src/Jackett/Utils/Clients/HttpWebClient.cs index 89e7eed01..5387dd932 100644 --- a/src/Jackett/Utils/Clients/HttpWebClient.cs +++ b/src/Jackett/Utils/Clients/HttpWebClient.cs @@ -1,110 +1,118 @@ -using AutoMapper; -using Jackett.Models; -using NLog; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; - -namespace Jackett.Utils.Clients -{ - class HttpWebClient : IWebClient - { - private Logger logger; - - public HttpWebClient(Logger l) - { - logger = l; - } - - - public void Init() - { - } - - public async Task GetBytes(WebRequest request) - { - logger.Debug(string.Format("WindowsWebClient:GetBytes(Url:{0})", request.Url)); - var result = await Run(request); - logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1} bytes", result.Status, (result.Content == null ? "" : result.Content.Length.ToString()))); - return result; - } - - public async Task GetString(WebRequest request) - { - logger.Debug(string.Format("WindowsWebClient:GetString(Url:{0})", request.Url)); - var result = await Run(request); - logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1}", result.Status, (result.Content == null ? "" : Encoding.UTF8.GetString(result.Content)))); - return Mapper.Map(result); - } - - private async Task Run(WebRequest request) - { - var cookies = new CookieContainer(); - if (!string.IsNullOrEmpty(request.Cookies)) - { - var uri = new Uri(request.Url); - foreach (var c in request.Cookies.Split(';')) - { - try - { - cookies.SetCookies(uri, c); - } - catch (CookieException ex) - { - logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message); - } - } - } - - var client = new HttpClient(new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. - UseCookies = true, - }); - - client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); - HttpResponseMessage response = null; - - if (request.Type == RequestType.POST) - { - var content = new FormUrlEncodedContent(request.PostData); - response = await client.PostAsync(request.Url, content); - } - else - { - response = await client.GetAsync(request.Url); - } - - var result = new WebClientByteResult(); - result.Content = await response.Content.ReadAsByteArrayAsync(); - if (response.Headers.Location != null) - { - result.RedirectingTo = response.Headers.Location.ToString(); - } - result.Status = response.StatusCode; - - // Compatiblity issue between the cookie format and httpclient - // Pull it out manually ignoring the expiry date then set it manually - // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer - IEnumerable cookieHeaders; - if (response.Headers.TryGetValues("set-cookie", out cookieHeaders)) - { - var cookieBuilder = new StringBuilder(); - foreach (var c in cookieHeaders) - { - cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.IndexOf(';') + 1)); - } - - result.Cookies = cookieBuilder.ToString().TrimEnd(); - } - - ServerUtil.ResureRedirectIsFullyQualified(request, result); - return result; - } - } -} +using AutoMapper; +using Jackett.Models; +using NLog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace Jackett.Utils.Clients +{ + public class HttpWebClient : IWebClient + { + private Logger logger; + + public HttpWebClient(Logger l) + { + logger = l; + } + + + public void Init() + { + } + + public async Task GetBytes(WebRequest request) + { + logger.Debug(string.Format("WindowsWebClient:GetBytes(Url:{0})", request.Url)); + var result = await Run(request); + logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1} bytes", result.Status, (result.Content == null ? "" : result.Content.Length.ToString()))); + return result; + } + + public async Task GetString(WebRequest request) + { + logger.Debug(string.Format("WindowsWebClient:GetString(Url:{0})", request.Url)); + var result = await Run(request); + logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1}", result.Status, (result.Content == null ? "" : Encoding.UTF8.GetString(result.Content)))); + return Mapper.Map(result); + } + + private async Task Run(WebRequest request) + { + var cookies = new CookieContainer(); + if (!string.IsNullOrEmpty(request.Cookies)) + { + var uri = new Uri(request.Url); + foreach (var c in request.Cookies.Split(';')) + { + try + { + cookies.SetCookies(uri, c); + } + catch (CookieException ex) + { + logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message); + } + } + } + + var client = new HttpClient(new HttpClientHandler + { + CookieContainer = cookies, + AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. + UseCookies = true, + }); + + client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); + HttpResponseMessage response = null; + + if (request.Headers != null) + { + foreach (var header in request.Headers) + { + client.DefaultRequestHeaders.Add(header.Key, header.Value); + } + } + + if (request.Type == RequestType.POST) + { + var content = new FormUrlEncodedContent(request.PostData); + response = await client.PostAsync(request.Url, content); + } + else + { + response = await client.GetAsync(request.Url); + } + + var result = new WebClientByteResult(); + result.Content = await response.Content.ReadAsByteArrayAsync(); + if (response.Headers.Location != null) + { + result.RedirectingTo = response.Headers.Location.ToString(); + } + result.Status = response.StatusCode; + + // Compatiblity issue between the cookie format and httpclient + // Pull it out manually ignoring the expiry date then set it manually + // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer + IEnumerable cookieHeaders; + if (response.Headers.TryGetValues("set-cookie", out cookieHeaders)) + { + var cookieBuilder = new StringBuilder(); + foreach (var c in cookieHeaders) + { + cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.IndexOf(';') + 1)); + } + + result.Cookies = cookieBuilder.ToString().TrimEnd(); + } + + ServerUtil.ResureRedirectIsFullyQualified(request, result); + return result; + } + } +} diff --git a/src/Jackett/Utils/Clients/WebRequest.cs b/src/Jackett/Utils/Clients/WebRequest.cs index 2d8862b18..a9ae22d6e 100644 --- a/src/Jackett/Utils/Clients/WebRequest.cs +++ b/src/Jackett/Utils/Clients/WebRequest.cs @@ -1,89 +1,95 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; - -namespace Jackett.Utils.Clients -{ - public class WebRequest - { - public WebRequest() - { - PostData = new List>(); - Type = RequestType.GET; - } - - public WebRequest(string url) - { - PostData = new List>(); - Type = RequestType.GET; - Url = url; - } - - public string Url { get; set; } - public IEnumerable> PostData { get; set; } - public string Cookies { get; set; } - public string Referer { get; set; } - public RequestType Type { get; set; } - - - public override bool Equals(System.Object obj) - { - if (obj is WebRequest) - { - var other = obj as WebRequest; - var postDataSame = PostData == null && other.PostData == null; - if (!postDataSame) - { - if (!(PostData == null || other.PostData == null)) - { - var ok = PostData.Count() == other.PostData.Count(); - foreach (var i in PostData) - { - if (!other.PostData.Any(item => item.Key == i.Key)) - { - ok = false; - break; - } - - if (PostData.FirstOrDefault(item => item.Key == i.Key).Value != other.PostData.FirstOrDefault(item => item.Key == i.Key).Value) - { - ok = false; - break; - } - } - - if (ok) - { - postDataSame = true; - } - } - } - - return other.Url == Url && - other.Referer == Referer && - other.Cookies == Cookies && - other.Type == Type && - postDataSame; - - } - else - { - return false; - } - } - - public override int GetHashCode() - { - return base.GetHashCode(); - } - } - - public enum RequestType - { - GET, - POST - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace Jackett.Utils.Clients +{ + public class WebRequest + { + public WebRequest() + { + PostData = new List>(); + Type = RequestType.GET; + Headers = new Dictionary(); + } + + public WebRequest(string url) + { + PostData = new List>(); + Type = RequestType.GET; + Url = url; + Headers = new Dictionary(); + } + + public string Url { get; set; } + public IEnumerable> PostData { get; set; } + public string Cookies { get; set; } + public string Referer { get; set; } + public RequestType Type { get; set; } + + /// + /// Warning this is only implemented on HTTPWebClient currently! + /// + public Dictionary Headers { get; set; } + + public override bool Equals(System.Object obj) + { + if (obj is WebRequest) + { + var other = obj as WebRequest; + var postDataSame = PostData == null && other.PostData == null; + if (!postDataSame) + { + if (!(PostData == null || other.PostData == null)) + { + var ok = PostData.Count() == other.PostData.Count(); + foreach (var i in PostData) + { + if (!other.PostData.Any(item => item.Key == i.Key)) + { + ok = false; + break; + } + + if (PostData.FirstOrDefault(item => item.Key == i.Key).Value != other.PostData.FirstOrDefault(item => item.Key == i.Key).Value) + { + ok = false; + break; + } + } + + if (ok) + { + postDataSame = true; + } + } + } + + return other.Url == Url && + other.Referer == Referer && + other.Cookies == Cookies && + other.Type == Type && + postDataSame; + + } + else + { + return false; + } + } + + public override int GetHashCode() + { + return base.GetHashCode(); + } + } + + public enum RequestType + { + GET, + POST + } +}