goreadability
goreadability copied to clipboard
Support web pages using non-utf8 charset
For example:
func bodyStr(res *http.Response) (s *string, err error) {
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, fmt.Errorf("ioutil.ReadAll failed: %v", err)
}
result := string(body)
_, cs, _ := charset.DetermineEncoding(body, res.Header.Get("Content-Type"))
if cs == "windows-1252" {
cs = "euc-kr"
}
if cs != "utf-8" {
result, err = iconv.ConvertString(result, cs, "utf-8")
if err != nil {
return nil, fmt.Errorf("Converting string from %v to utf-8 failed, url: %v",
cs, res.Request.URL.String())
}
}
return &result, nil
}