icap-js-injection-example.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500
  1. /*
  2. An example of how to use go-icap.
  3. Run this program and Squid on the same machine.
  4. Put the following lines in squid.conf:
  5. acl GET method GET
  6. icap_enable on
  7. icap_service service_req reqmod_precache icap://127.0.0.1:1344/injectjs/
  8. adaptation_access service_req allow GET
  9. adaptation_access service_req deny all
  10. (The ICAP server needs to be started before Squid is.)
  11. Set your browser to use the Squid proxy.
  12. Some Refrences:
  13. - https://groups.google.com/forum/#!topic/golang-nuts/J-Y4LtdGNSw
  14. */
  15. package main
  16. import (
  17. "bytes"
  18. "compress/gzip"
  19. "errors"
  20. "flag"
  21. "fmt"
  22. "io"
  23. "io/ioutil"
  24. "log"
  25. "net/http"
  26. "os"
  27. "reflect"
  28. "strconv"
  29. "strings"
  30. "time"
  31. "github.com/asaskevich/govalidator"
  32. "github.com/elico/icap"
  33. "github.com/patrickmn/go-cache"
  34. "gopkg.in/redis.v3"
  35. )
  36. var (
  37. isTag = "HTML-JS-Injector"
  38. debug *bool
  39. address *string
  40. maxConnections *string
  41. redisAddress *string
  42. fullOverride = false
  43. upnkeyTimeout *int
  44. useGoCache *bool
  45. err error
  46. letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
  47. )
  48. var goCacheLocal *cache.Cache
  49. var redisDB redis.Client
  50. // GlobalHTTPClient --
  51. var GlobalHTTPClient *http.Client
  52. func overrideExists(req *icap.Request) bool {
  53. if *debug {
  54. fmt.Println("Checking Override")
  55. }
  56. if fullOverride {
  57. if *debug {
  58. fmt.Println("Full Override active")
  59. }
  60. return true
  61. }
  62. if _, acceptExists := req.Request.Header["Accept"]; acceptExists {
  63. if *debug {
  64. fmt.Println(req.Request.Header["Accept"][0])
  65. }
  66. if strings.Contains(req.Request.Header["Accept"][0], "MoreCache/Override") {
  67. if *debug {
  68. fmt.Println("Override true")
  69. }
  70. return true
  71. }
  72. }
  73. if *debug {
  74. fmt.Println("Override false")
  75. }
  76. return false
  77. }
  78. func noCache(req *icap.Request) bool {
  79. if *debug {
  80. fmt.Println("Checking Request or response for \"no-cache\" => ")
  81. }
  82. if _, cacheControlExists := req.Request.Header["Cache-Control"]; cacheControlExists {
  83. if *debug {
  84. fmt.Println("Cache-Control Exists in the request: ")
  85. fmt.Println(req.Request.Header["Cache-Control"][0])
  86. }
  87. if strings.Contains(req.Request.Header["Cache-Control"][0], "no-cache") {
  88. if *debug {
  89. fmt.Println("\"no-cache\" Exists in the request!")
  90. }
  91. return true
  92. }
  93. }
  94. if _, cacheControlExists := req.Response.Header["Cache-Control"]; cacheControlExists {
  95. if *debug {
  96. fmt.Println("Cache-Control Exists in the response: ")
  97. fmt.Println("Cache-Control Header =>", reflect.TypeOf(req.Response.Header["Cache-Control"]))
  98. fmt.Println("Reflect tpyeof Cache-Control Header =>", reflect.TypeOf(req.Response.Header["Cache-Control"]))
  99. fmt.Println("len Cache-Control Header =>", len(req.Response.Header["Cache-Control"]))
  100. fmt.Println(req.Response.Header["Cache-Control"])
  101. }
  102. if len(req.Response.Header["Cache-Control"]) > 0 && strings.Contains(strings.Join(req.Response.Header["Cache-Control"], ", "), "no-cache") {
  103. fmt.Println("\"no-cache\" Exists in the response!")
  104. return true
  105. }
  106. }
  107. if *debug {
  108. fmt.Println("Cache-Control headers Doesn't Exists in this requset and response")
  109. }
  110. return false
  111. }
  112. func wrongMethod(req *icap.Request) bool {
  113. if *debug {
  114. fmt.Println("Checking Request method => ", req.Request.Method, req.Request.URL.String())
  115. }
  116. if req.Request.Method == "GET" {
  117. return false
  118. }
  119. return true
  120. }
  121. func htmlJSInject(w icap.ResponseWriter, req *icap.Request) {
  122. localDebug := false
  123. useTProxy := false
  124. if strings.Contains(req.URL.RawQuery, "debug=1") {
  125. localDebug = true
  126. }
  127. if strings.Contains(req.URL.RawQuery, "tproxy=1") {
  128. useTProxy = true
  129. }
  130. h := w.Header()
  131. h.Set("isTag", isTag)
  132. h.Set("Service", "HTML JS Injector ICAP serivce")
  133. if *debug {
  134. fmt.Fprintln(os.Stderr, "Printing the full ICAP request")
  135. fmt.Fprintln(os.Stderr, req)
  136. fmt.Fprintln(os.Stderr, req.Request)
  137. fmt.Fprintln(os.Stderr, req.Response)
  138. }
  139. switch req.Method {
  140. case "OPTIONS":
  141. h.Set("Methods", "REQMOD, RESPMOD")
  142. h.Set("Options-TTL", "1800")
  143. h.Set("Allow", "204, 206")
  144. h.Set("Preview", "0")
  145. h.Set("Transfer-Preview", "*")
  146. h.Set("Max-Connections", *maxConnections)
  147. h.Set("X-Include", "X-Client-IP, X-Authenticated-Groups, X-Authenticated-User, X-Subscriber-Id, X-Server-Ip, X-Store-Id")
  148. w.WriteHeader(200, nil, false)
  149. case "REQMOD":
  150. modified := false
  151. nullBody := false
  152. allow206 := false
  153. allow204 := false
  154. hasClientIPHeader := false
  155. if _, allow204Exists := req.Header["Allow"]; allow204Exists {
  156. if strings.Contains(req.Header["Allow"][0], "204") {
  157. allow204 = true
  158. }
  159. }
  160. if *debug || localDebug {
  161. for k, v := range req.Header {
  162. fmt.Fprintln(os.Stderr, "The ICAP headers:")
  163. fmt.Fprintln(os.Stderr, "key size:", len(req.Header[k]))
  164. fmt.Fprintln(os.Stderr, "key:", k, "value:", v)
  165. }
  166. }
  167. xClientIP := req.Header.Get("X-Client-IP")
  168. if govalidator.IsIP(xClientIP) {
  169. hasClientIPHeader = true
  170. }
  171. if hasClientIPHeader && (*debug || localDebug) {
  172. fmt.Fprintln(os.Stderr, "IP:", xClientIP, "Requested-URL:", req.Request.URL.String())
  173. }
  174. if _, encapsulationExists := req.Header["Encapsulated"]; encapsulationExists {
  175. if strings.Contains(req.Header["Encapsulated"][0], "null-body=") {
  176. nullBody = true
  177. }
  178. }
  179. if _, allow206Exists := req.Header["Allow"]; allow206Exists {
  180. if strings.Contains(req.Header["Allow"][0], "206") {
  181. allow206 = true
  182. }
  183. }
  184. _, _, _, _ = nullBody, allow206, modified, allow204
  185. if wrongMethod(req) {
  186. if *debug {
  187. fmt.Println("This request has a", req.Request.Method, "method which is not being analyzed")
  188. }
  189. w.WriteHeader(204, nil, false)
  190. return
  191. }
  192. if *debug || localDebug {
  193. for k, v := range req.Request.Header {
  194. fmt.Fprintln(os.Stderr, "key:", k, "value:", v)
  195. }
  196. }
  197. if strings.HasPrefix(req.Request.URL.String(), "http://") {
  198. if *debug {
  199. fmt.Println("XYZ HTTP url match:", req.Request.URL.String())
  200. }
  201. var client *http.Client
  202. if useTProxy && govalidator.IsIP(xClientIP) {
  203. tproxyClient := GlobalHTTPClients[xClientIP]
  204. if tproxyClient == nil {
  205. client = CreateTproxyHTTPClient(xClientIP)
  206. }
  207. } else {
  208. client = GlobalHTTPClient
  209. }
  210. newReq, err := http.NewRequest(req.Request.Method, req.Request.URL.String(), nil)
  211. newReq.Header = req.Request.Header
  212. switch {
  213. case newReq.Header.Get("Accept-Encoding") == "gzip":
  214. newReq.Header.Del("Accept-Encoding")
  215. if *debug {
  216. fmt.Println(req.Request.URL.String(), "Removed Accept-Encoding Header since it contains only gzip")
  217. }
  218. case strings.Contains(newReq.Header.Get("Accept-Encoding"), "gzip, "):
  219. newReq.Header.Set("Accept-Encoding", strings.Replace(newReq.Header.Get("Accept-Encoding"), "gzip, ", "", -1))
  220. if *debug {
  221. fmt.Println(req.Request.URL.String(), "Removed \"gzip, \" From Accept-Encoding Header")
  222. }
  223. case strings.Contains(newReq.Header.Get("Accept-Encoding"), ", gzip"):
  224. newReq.Header.Set("Accept-Encoding", strings.Replace(newReq.Header.Get("Accept-Encoding"), ", gzip", "", -1))
  225. if *debug {
  226. fmt.Println(req.Request.URL.String(), "Removed \", gzip\" From Accept-Encoding Header")
  227. }
  228. default:
  229. if *debug {
  230. fmt.Println(req.Request.URL.String(), "No gzip In Accept-Encoding Header")
  231. }
  232. // no gzip
  233. }
  234. switch {
  235. case newReq.Header.Get("Accept-Encoding") == "deflate":
  236. newReq.Header.Del("Accept-Encoding")
  237. if *debug {
  238. fmt.Println(req.Request.URL.String(), "Removed Accept-Encoding Header since it contains only deflate")
  239. }
  240. case strings.Contains(newReq.Header.Get("Accept-Encoding"), "deflate, "):
  241. newReq.Header.Set("Accept-Encoding", strings.Replace(newReq.Header.Get("Accept-Encoding"), "deflate, ", "", -1))
  242. if *debug {
  243. fmt.Println(req.Request.URL.String(), "Removed \"deflate, \" From Accept-Encoding Header")
  244. }
  245. case strings.Contains(newReq.Header.Get("Accept-Encoding"), ", deflate"):
  246. newReq.Header.Set("Accept-Encoding", strings.Replace(newReq.Header.Get("Accept-Encoding"), ", deflate", "", -1))
  247. if *debug {
  248. fmt.Println(req.Request.URL.String(), "Removed \", deflate\" From Accept-Encoding Header")
  249. }
  250. default:
  251. if *debug {
  252. fmt.Println(req.Request.URL.String(), "No deflate In Accept-Encoding Header")
  253. }
  254. // no gzip
  255. }
  256. switch {
  257. case newReq.Header.Get("Accept-Encoding") == "sdch":
  258. newReq.Header.Del("Accept-Encoding")
  259. if *debug {
  260. fmt.Println(req.Request.URL.String(), "Removed Accept-Encoding Header since it contains only sdch")
  261. }
  262. case strings.Contains(newReq.Header.Get("Accept-Encoding"), "sdch, "):
  263. newReq.Header.Set("Accept-Encoding", strings.Replace(newReq.Header.Get("Accept-Encoding"), "sdch, ", "", -1))
  264. if *debug {
  265. fmt.Println(req.Request.URL.String(), "Removed \"sdch, \" From Accept-Encoding Header")
  266. }
  267. case strings.Contains(newReq.Header.Get("Accept-Encoding"), ", sdch"):
  268. newReq.Header.Set("Accept-Encoding", strings.Replace(newReq.Header.Get("Accept-Encoding"), ", sdch", "", -1))
  269. if *debug {
  270. fmt.Println(req.Request.URL.String(), "Removed \", sdch\" From Accept-Encoding Header")
  271. }
  272. default:
  273. if *debug {
  274. fmt.Println(req.Request.URL.String(), "No sdch In Accept-Encoding Header")
  275. }
  276. // no gzip
  277. }
  278. switch {
  279. case newReq.Header.Get("Accept-Encoding") == "br":
  280. newReq.Header.Del("Accept-Encoding")
  281. if *debug {
  282. fmt.Println(req.Request.URL.String(), "Removed Accept-Encoding Header since it contains only br")
  283. }
  284. case strings.Contains(newReq.Header.Get("Accept-Encoding"), "br, "):
  285. newReq.Header.Set("Accept-Encoding", strings.Replace(newReq.Header.Get("Accept-Encoding"), "br, ", "", -1))
  286. if *debug {
  287. fmt.Println(req.Request.URL.String(), "Removed \"br, \" From Accept-Encoding Header")
  288. }
  289. case strings.Contains(newReq.Header.Get("Accept-Encoding"), ", br"):
  290. newReq.Header.Set("Accept-Encoding", strings.Replace(newReq.Header.Get("Accept-Encoding"), ", br", "", -1))
  291. if *debug {
  292. fmt.Println(req.Request.URL.String(), "Removed \", br\" From Accept-Encoding Header")
  293. }
  294. default:
  295. if *debug {
  296. fmt.Println(req.Request.URL.String(), "No rb In Accept-Encoding Header")
  297. }
  298. // no gzip
  299. }
  300. if len(newReq.Header.Get("Accept-Encoding")) < 1 {
  301. newReq.Header.Del("Accept-Encoding")
  302. if *debug {
  303. fmt.Println(req.Request.URL.String(), "Deleteing Accept-Encoding Header since it's empty:", newReq.Header.Get("Accept-Encoding"))
  304. }
  305. }
  306. if *debug {
  307. fmt.Println(req.Request.URL.String(), "Accept-Encoding Header:", newReq.Header.Get("Accept-Encoding"))
  308. }
  309. originalResp, err := client.Do(newReq)
  310. if err != nil {
  311. if *debug {
  312. fmt.Println(err)
  313. }
  314. w.WriteHeader(204, nil, false)
  315. return
  316. }
  317. resp := new(http.Response)
  318. resp.Status = originalResp.Status
  319. resp.StatusCode = originalResp.StatusCode
  320. resp.Proto = originalResp.Proto
  321. resp.ProtoMajor = originalResp.ProtoMajor
  322. resp.ProtoMinor = originalResp.ProtoMinor
  323. resp.Header = originalResp.Header
  324. if *debug {
  325. fmt.Println(req.Request.URL.String(), "Response Header:", resp.Header)
  326. }
  327. resp.Request = originalResp.Request
  328. // What if it is a CONNECT request, .. shouldn't happen
  329. content, err := ioutil.ReadAll(originalResp.Body)
  330. if err != nil {
  331. if *debug {
  332. fmt.Println("returning 204 response due to an Error reading Body response fore request:", resp.Request)
  333. }
  334. w.WriteHeader(204, nil, false)
  335. return
  336. }
  337. pageContent := ""
  338. if originalResp.Header.Get("Content-Encoding") == "gzip" || strings.Contains(originalResp.Header.Get("Content-Encoding"), "gzip ") || strings.Contains(originalResp.Header.Get("Content-Encoding"), ",gzip") {
  339. rdata := bytes.NewReader(content)
  340. r, _ := gzip.NewReader(rdata)
  341. s, _ := ioutil.ReadAll(r)
  342. pageContent = string(s)
  343. } else {
  344. pageContent = string(content)
  345. }
  346. if len(pageContent) > 0 {
  347. //resp.Body = ioutil.NopCloser(bytes.NewBufferString(pageContent))
  348. w.WriteHeader(200, resp, true)
  349. io.WriteString(w, pageContent)
  350. } else {
  351. w.WriteHeader(200, resp, false)
  352. }
  353. return
  354. }
  355. // What I have been using for captive portal
  356. //io.WriteString(w, challengePage)
  357. if *debug {
  358. fmt.Println("end of the line 204 response!.. Shouldn't happen.")
  359. }
  360. w.WriteHeader(204, nil, false)
  361. return
  362. case "RESPMOD":
  363. w.WriteHeader(204, nil, false)
  364. return
  365. default:
  366. w.WriteHeader(405, nil, false)
  367. if *debug || localDebug {
  368. fmt.Fprintln(os.Stderr, "Invalid request method")
  369. }
  370. }
  371. }
  372. func defaultIcap(w icap.ResponseWriter, req *icap.Request) {
  373. localDebug := false
  374. if strings.Contains(req.URL.RawQuery, "debug=1") {
  375. localDebug = true
  376. }
  377. h := w.Header()
  378. h.Set("isTag", isTag)
  379. h.Set("Service", "YouTube GoogleVideo Predictor ICAP serivce")
  380. if *debug || localDebug {
  381. fmt.Fprintln(os.Stderr, "Printing the full ICAP request")
  382. fmt.Fprintln(os.Stderr, req)
  383. fmt.Fprintln(os.Stderr, req.Request)
  384. }
  385. switch req.Method {
  386. case "OPTIONS":
  387. h.Set("Methods", "REQMOD, RESPMOD")
  388. h.Set("Options-TTL", "1800")
  389. h.Set("Allow", "204")
  390. h.Set("Preview", "0")
  391. h.Set("Transfer-Preview", "*")
  392. h.Set("Max-Connections", *maxConnections)
  393. h.Set("This-Server", "Default ICAP url which bypass all requests adaptation")
  394. h.Set("X-Include", "X-Client-IP, X-Authenticated-Groups, X-Authenticated-User, X-Subscriber-Id, X-Server-Ip, X-Store-Id")
  395. w.WriteHeader(200, nil, false)
  396. case "REQMOD":
  397. if *debug || localDebug {
  398. fmt.Fprintln(os.Stderr, "Default REQMOD, you should use the apropriate ICAP service URL")
  399. }
  400. w.WriteHeader(204, nil, false)
  401. case "RESPMOD":
  402. if *debug || localDebug {
  403. fmt.Fprintln(os.Stderr, "Default RESPMOD, you should use the apropriate ICAP service URL")
  404. }
  405. w.WriteHeader(204, nil, false)
  406. default:
  407. w.WriteHeader(405, nil, false)
  408. if *debug || localDebug {
  409. fmt.Fprintln(os.Stderr, "Invalid request method")
  410. }
  411. }
  412. }
  413. func init() {
  414. fmt.Fprintln(os.Stderr, "Starting YouTube GoogleVideo Predictor ICAP serivce")
  415. debug = flag.Bool("d", false, "Debug mode can be \"1\" or \"0\" for no")
  416. address = flag.String("p", "127.0.0.1:1344", "Listening address for the ICAP service")
  417. maxConnections = flag.String("maxcon", "4000", "Maximum number of connections for the ICAP service")
  418. redisAddress = flag.String("redis-address", "127.0.0.1:6379", "Redis DB address to store youtube tokens")
  419. upnkeyTimeout = flag.Int("cache-key-timeout", 360, "Redis or GoCache DB key timeout in Minutes")
  420. useGoCache = flag.Bool("Use GoCache", true, "GoCache DB is used by default and if disabled then Redis is used")
  421. flag.Parse()
  422. }
  423. func main() {
  424. fmt.Fprintln(os.Stderr, "running YouTube GoogleVideo Predictor ICAP serivce :D")
  425. if *debug {
  426. fmt.Fprintln(os.Stderr, "Config Variables:")
  427. fmt.Fprintln(os.Stderr, "Debug: => "+strconv.FormatBool(*debug))
  428. fmt.Fprintln(os.Stderr, "Listen Address: => "+*address)
  429. fmt.Fprintln(os.Stderr, "Redis DB Address: => "+*redisAddress)
  430. fmt.Fprintln(os.Stderr, "Maximum number of Connections: => "+*maxConnections)
  431. }
  432. if *useGoCache {
  433. goCacheLocal = cache.New(time.Duration(*upnkeyTimeout)*time.Minute, 10*time.Minute)
  434. } else {
  435. redisDB = *redis.NewClient(&redis.Options{
  436. Addr: *redisAddress,
  437. Password: "", // no password set
  438. DB: 0, // use default DB
  439. })
  440. }
  441. GlobalHTTPClient = &http.Client{}
  442. GlobalHTTPClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
  443. return errors.New("redirect")
  444. }
  445. icap.HandleFunc("/injectjs/", htmlJSInject)
  446. icap.HandleFunc("/", defaultIcap)
  447. log.Fatal(icap.ListenAndServe(*address, nil))
  448. }