stjet/koxinga
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

POST forms, cookies, parsing fix?

Browse Source 
Some logins and posting operations, like certain simple forums are working. Others, like wikimedia, are still not :(
This commit is contained in:
stjet
2026-07-17 05:20:15 +00:00
parent 2d7030f5f4
commit 264ac04b84
8 changed files with 2270 additions and 918 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@@ -560,7 +560,7 @@ dependencies = [
[[package]]
name = "koxinga"
version = "1.0.0"
version = "1.1.0-rc.0"
dependencies = [
"ming-wm-lib",
"reqwest",

2
Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "koxinga"
version = "1.0.0"
version = "1.1.0-rc.0"
edition = "2021"
[lints.clippy]

2
real_tests/hn_user.html Normal file
View File

@@ -0,0 +1,2 @@
<html lang="en" op="user"><head><meta name="referrer" content="origin"><meta name="viewport" content="width=device-width, initial-scale=1.0"><link rel="stylesheet" type="text/css" href="news.css?rbDKZB3Gz3kvhIuyRiN1"><link rel="icon" href="y18.svg"><link rel="canonical" href="https://news.ycombinator.com/user?id=panza"><title>Profile: panza | Hacker News</title></head><body><center><table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%" bgcolor="#f6f6ef"><tr><td bgcolor="#ff6600"><table border="0" cellpadding="0" cellspacing="0" width="100%" style="padding:2px"><tr><td style="width:18px;padding-right:4px"><a href="https://news.ycombinator.com"><img src="y18.svg" width="18" height="18" style="border:1px white solid; display:block"></a></td><td style="line-height:12pt; height:10px;"><span class="pagetop"><b class="hnname"><a href="news">Hacker News</a></b><a href="newest">new</a> | <a href="front">past</a> | <a href="newcomments">comments</a> | <a href="ask">ask</a> | <a href="show">show</a> | <a href="jobs">jobs</a> | <a href="submit" rel="nofollow">submit</a></span></td><td style="text-align:right;padding-right:4px;"><span class="pagetop"><a href="login?goto=user%3Fid%3Dpanza">login</a></span></td></tr></table></td></tr><tr style='height:10px'/><tr id="bigbox"><td><table border="0"><tr class="athing"><td valign="top">user:</td><td timestamp="1289714067"><a href="user?id=panza" class="hnuser">panza</a></td></tr><tr><td valign="top">created:</td><td><span class="age"><a href="front?day=2010-11-14&birth=panza">November 14, 2010</a></span></td></tr><tr><td valign="top">karma:</td><td>482</td></tr><tr><td valign="top">about:</td><td style="overflow:hidden"></td></tr><tr><td></td><td><a href="submitted?id=panza"><u>submissions</u></a></td></tr><tr><td></td><td><a href="threads?id=panza"><u>comments</u></a></td></tr><tr><td></td><td><a href="favorites?id=panza"><u>favorites</u></a></td></tr></table><br><br>
</td></tr></table></center></body><script type="text/javascript" src="hn.js?rbDKZB3Gz3kvhIuyRiN1"></script></html>

2883
real_tests/wikipedia.html
View File

File diff suppressed because one or more lines are too long

86
src/http.rs
View File

@@ -1,29 +1,103 @@
use std::collections::HashMap;
//use ming_wm_lib::logging::log;
use ming_wm_lib::utils::get_rest_of_split;
use crate::url::Url;
use reqwest::blocking::Client;
//for now, just a thin wrapper
pub struct HttpClient {
client: Client,
no_redirect_client: Client,
}
impl std::default::Default for HttpClient {
fn default() -> Self {
//for privacy can change to more common one
let client = Client::builder().user_agent("Koxinga").build().unwrap();
//we lie cause otherwise people block us. can't be honest no more
let client = Client::builder().user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3").build().unwrap();
let no_redirect_client = Client::builder().user_agent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3").redirect(reqwest::redirect::Policy::none()).build().unwrap();
Self {
client,
no_redirect_client,
}
}
}
fn serialise_cookies(cookies: &HashMap<String, String>) -> String {
let mut c_header = String::new();
for (name, value) in cookies {
c_header += &format!("{}{}={}", if !c_header.is_empty() {
"; "
} else {
""
}, name, value);
}
c_header
}
impl HttpClient {
pub fn get(&self, url: &str) -> Option<String> {
if let Ok(resp) = self.client.get(url).send() {
//the second return value, the final url, may differ from the input url, because of redirects
pub fn get(&self, url: &str, cookies: Option<&HashMap<String, String>>) -> Option<(String, String)> {
let mut req = self.client.get(url);
//nom nom nom
if let Some(cookies) = cookies {
let c_header = serialise_cookies(cookies);
//set cookie header
if !c_header.is_empty() {
req = req.header("Cookie", c_header);
}
}
if let Ok(resp) = req.send() {
let final_url = resp.url().as_str().to_string();
if let Ok(text) = resp.text() {
return Some(text);
return Some((text, final_url));
}
}
None
}
//todo: form submit (get/post)
//todo: POST for form submit for cookies
pub fn post(&self, url: Url, body: String, from_url: Url, cookies: Option<&HashMap<String, String>>) -> Option<(Url, Vec<(String, String)>)> {
let mut url = url;
let mut req = self.no_redirect_client.post(url.to_string()).body(body).header("Content-Type", "application/x-www-form-urlencoded").header("Origin", format!("https://{}", from_url.hostname));
if let Some(cookies) = cookies {
let c_header = serialise_cookies(cookies);
if !c_header.is_empty() {
req = req.header("Cookie", c_header);
}
}
let mut cookies: Vec<(String, String)> = Vec::new();
let mut redirect_count = 0;
loop {
if redirect_count > 5 {
break;
}
if let Ok(resp) = req.send() {
let c_headers = resp.headers().get_all("Set-Cookie");
for header in c_headers {
if let Ok(value) = header.to_str() {
let mut parts = value.split(";").next().unwrap().split("=");
let name = parts.next().unwrap_or_default().to_string();
let value = get_rest_of_split(&mut parts, Some("="));
cookies.push((name, value));
}
}
if resp.status().is_redirection() {
redirect_count += 1;
//follow location resp header
if let Some(location) = resp.headers().get("Location") {
url = Url::new_maybe_relative(location.to_str().unwrap_or_default().to_string(), url);
req = self.no_redirect_client.get(url.to_string());
continue;
}
}
return Some((url, cookies)); //break out
} else {
break;
}
}
None
}
}

87
src/main.rs
View File

@@ -1,3 +1,5 @@
//TODO: RUN CLIPPY LINT
use std::vec::Vec;
use std::vec;
use std::fmt;
@@ -16,7 +18,7 @@ use ming_wm_lib::ipc::listen;
mod http;
use crate::http::HttpClient;
mod xml;
use crate::xml::{ parse, remove_quotes, Form, FormSubmitMethod, Node, OutputType };
use crate::xml::{ parse, remove_quotes, handle_escaped, Form, FormSubmitMethod, Node, OutputType, REPLACE, URL_REPLACE };
mod url;
use crate::url::Url;
@@ -30,7 +32,7 @@ enum State {
Maybeg,
}
#[derive(Default, PartialEq)]
#[derive(Default, PartialEq, Clone, Copy)]
enum Mode {
#[default]
Normal,
@@ -88,6 +90,7 @@ struct KoxingaBrowser {
fonts: Vec<String>,
mode: Mode,
state: State,
cookies: HashMap<String, HashMap<String, String>>, //cookies for each site
max_lines: usize,
top_line_no: usize,
url: Option<Url>,
@@ -219,9 +222,16 @@ impl WindowLike for KoxingaBrowser {
url
} else {
//if Mode::Url
Url::new(self.input.clone())
//check if starts with http:// or https://
let url = Url::new(self.input.clone());
if !url.valid_scheme {
Url::new(format!("https://lite.duckduckgo.com/lite?q={}", self.input))
} else {
url
}
};
if let Some(text) = self.client.get(&new_url.to_string()) {
if let Some((text, new_new_url)) = self.client.get(&new_url.to_string(), self.cookies.get(&new_url.hostname)) {
let new_url = Url::new(new_new_url);
self.change_url(new_url, text);
WindowMessageResponse::JustRedraw
} else {
@@ -254,18 +264,23 @@ impl WindowLike for KoxingaBrowser {
let form_index = self.input.parse::<usize>().unwrap();
if form_index < self.forms.len() {
let form_info = &self.forms[form_index];
let form_url = if let Some(action) = &form_info.action {
Url::new_maybe_relative(action.to_string(), self.url.clone().unwrap())
} else {
self.url.clone().unwrap()
};
match form_info.method {
FormSubmitMethod::Get => {
//construct url to redirect to
let mut form_url = Url::new_maybe_relative(form_info.action.clone(), self.url.clone().unwrap());
let mut form_url = form_url;
//key aka name attr
for key in &form_info.input_names {
if let Some(value) = self.form_inputs.get(&(form_index, key.clone())) {
form_url.append_query(&key, value);
}
}
//log(&format!("{}", form_url.clone()));
if let Some(text) = self.client.get(&form_url.to_string()) {
if let Some((text, new_new_url)) = self.client.get(&form_url.to_string(), self.cookies.get(&form_url.hostname)) {
let form_url = Url::new(new_new_url);
self.change_url(form_url, text);
WindowMessageResponse::JustRedraw
} else {
@@ -274,8 +289,30 @@ impl WindowLike for KoxingaBrowser {
},
FormSubmitMethod::Post => {
//todo. maybe later
//
WindowMessageResponse::DoNothing
let mut body = String::new();
for key in &form_info.input_names {
if let Some(value) = self.form_inputs.get(&(form_index, key.clone())) {
body += &format!("{}{}={}", if body.len() > 0 { "&" } else { "" }, key, handle_escaped(&handle_escaped(value, REPLACE.to_vec(), false), URL_REPLACE.to_vec(), true).replace(" ", "+"));
}
}
let post_cookies = self.cookies.get(&form_url.hostname);
if let Some((new_url, cookies)) = self.client.post(form_url, body, self.url.clone().unwrap(), post_cookies) {
//add to cookies
for cookie in cookies {
//todo: replace old cookie with same name
let hostname = self.url.clone().unwrap().hostname;
if !self.cookies.contains_key(&hostname) {
self.cookies.insert(hostname.clone(), HashMap::new());
}
self.cookies.get_mut(&hostname).unwrap().insert(cookie.0, cookie.1);
}
if let Some((text, new_url)) = self.client.get(&new_url.to_string(), self.cookies.get(&new_url.hostname)) {
let new_url = Url::new(new_url);
self.change_url(new_url, text);
}
}
self.mode = Mode::Normal;
WindowMessageResponse::JustRedraw
},
}
} else {
@@ -297,9 +334,10 @@ impl WindowLike for KoxingaBrowser {
WindowMessageResponse::DoNothing
}
} else if key_press.is_escape() {
self.mode = Mode::Normal;
self.input = String::new();
if self.mode == Mode::Link || self.mode == Mode::FormSubmit || self.mode == Mode::FormInput {
let old_mode = self.mode;
self.mode = Mode::Normal;
if old_mode == Mode::Link || old_mode == Mode::FormSubmit || old_mode == Mode::FormInput {
self.calc_page(false);
}
WindowMessageResponse::JustRedraw
@@ -315,6 +353,14 @@ impl WindowLike for KoxingaBrowser {
},
}
},
WindowMessage::CtrlKeyPress(key_press) => {
if key_press.key == 'a' {
self.input = String::new();
WindowMessageResponse::JustRedraw
} else {
WindowMessageResponse::DoNothing
}
},
_ => WindowMessageResponse::DoNothing,
}
}
@@ -387,7 +433,7 @@ impl KoxingaBrowser {
}
pub fn change_url(&mut self, new_url: Url, text: String) {
self.url = Some(new_url);
self.url = Some(new_url.clone());
self.top_line_no = 0;
self.top_level_nodes = parse(&text);
self.input = String::new();
@@ -420,6 +466,11 @@ impl KoxingaBrowser {
break;
}
}
//handle if no <body> tag (wtf wikimedia error page)
if outputs.is_empty() {
//hey, why not at that point...
outputs = self.top_level_nodes[html_index].to_output();
}
}
}
let mut y = 2;
@@ -468,15 +519,15 @@ impl KoxingaBrowser {
} + "Submit Form";
form_counter += 1;
Some(t)
} else if let OutputType::TextInput(name) = &o {
} else if let OutputType::TextInput(name, default_value) = &o {
subtype = Subtype::TextInput;
if new_page {
self.form_inputs.insert((form_counter, name.to_string()), String::new());
self.form_inputs.insert((form_counter, name.to_string()), default_value.to_string());
}
let t = if self.mode == Mode::FormInput || self.mode == Mode::FormSubmit {
format!("{},{}={}", form_counter.to_string(), name, self.form_inputs.get(&(form_counter, name.to_owned())).unwrap())
format!("{},{}={}\n", form_counter.to_string(), name, self.form_inputs.get(&(form_counter, name.to_owned())).unwrap())
} else {
name.to_owned()
name.to_owned() + "\n"
};
Some(t)
} else {
@@ -491,7 +542,7 @@ impl KoxingaBrowser {
let mut start_x = x;
for c in s.chars() {
let c_width = measure_text_with_cache(&mut fc_getter, &self.fonts, &c.to_string(), None).width + 1; //+1 for horiz spacing
if x + c_width > self.dimensions[0] {
if x + c_width > self.dimensions[0] || c == '\n' {
//full line, add draw instruction
self.page.push((start_x, y, line, subtype));
line = String::new();
@@ -500,9 +551,11 @@ impl KoxingaBrowser {
y += LINE_HEIGHT;
line_count += 1;
}
if c != '\n' {
line += &c.to_string();
x += c_width;
}
}
if line.len() > 0 {
self.page.push((start_x, y, line, subtype));
}

10
src/url.rs
View File

@@ -1,11 +1,14 @@
use std::vec::Vec;
use std::fmt;
const VALID_SCHEMES: [&'static str; 2] = ["HTTP", "HTTPS"]; //more to come in future?? who knows
//for the moment, we don't care about query params or fragments and the like
#[derive(Clone)]
pub struct Url {
scheme: String, //http or https, probably
hostname: String,
pub valid_scheme: bool,
pub hostname: String,
path: Vec<String>,
query: Option<String>, //empty or somethign like ?value1=yes&value2=abcd
}
@@ -24,14 +27,15 @@ impl Url {
let mut queries = url.split("?");
let mut p = queries.next().unwrap().split("://");
let scheme = p.next().unwrap_or("").to_string();
let valid_scheme = VALID_SCHEMES.contains(&scheme.to_uppercase().as_str());
p = p.next().unwrap_or("").split("/");
let hostname = p.next().unwrap_or("").to_string();
let path = p.filter(|s| *s != "").map(|s| s.to_string()).collect();
let query = match queries.next() {
Some(q) => Some(q.to_string()),
Some(q) => Some(format!("?{}", q)),
None => None,
};
Self { scheme, hostname, path, query }
Self { scheme, valid_scheme, hostname, path, query }
}
pub fn new_maybe_relative(url: String, current_url: Url) -> Url {

92
src/xml.rs
View File

@@ -15,23 +15,43 @@ const SELF_CLOSING: [&'static str; 9] = ["link", "meta", "input", "img", "br", "
//not all of them, eg there is intentionally no div
const BLOCK_LEVEL: [&'static str; 13] = ["p", "br", "li", "tr", "header", "footer", "section", "h1", "h2", "h3", "h4", "h5", "h6"];
const REPLACE: [(&'static str, &'static str); 6] = [
pub const REPLACE: [(&'static str, &'static str); 7] = [
("&nbsp;", " "),
("&#x27;", "'"),
("&quot;", "\""),
("&#x2F;", "/"),
("&gt;", ">"),
("&lt;", "<"),
("&amp;", "&"),
];
pub const URL_REPLACE: [(&'static str, &'static str); 12] = [
("%22", "\""),
("%2B", "+"),
("%2C", ","),
("%2D", "-"),
("%2F", "/"),
("%3A", ":"),
("%5C", "\\"),
("%5B", "["),
("%5D", "]"),
("%5F", "_"),
("%7B", "{"),
("%7D", "}"),
];
fn is_whitespace(c: char) -> bool {
c == ' ' || c == '\x09'
}
fn handle_escaped(s: &str) -> String {
pub fn handle_escaped(s: &str, replace_list: Vec<(&str, &str)>, inverse: bool) -> String {
let mut s = s.to_string();
for rp in REPLACE {
for rp in replace_list {
if !inverse {
s = s.replace(rp.0, rp.1);
} else {
s = s.replace(rp.1, rp.0);
}
}
s
}
@@ -54,7 +74,7 @@ pub enum FormSubmitMethod {
#[derive(Debug, PartialEq, Clone)]
pub struct Form {
pub action: String, //url
pub action: Option<String>, //url, if None, defaults to same url
pub method: FormSubmitMethod,
pub input_names: Vec<String>,
}
@@ -67,7 +87,7 @@ pub enum OutputType {
Newline,
//only support one per line, once indented, will keep being indented until overriden, for now
Indent(usize),
TextInput(String),
TextInput(String, String), //name, default value
Form(Form),
}
@@ -89,7 +109,7 @@ impl Node {
if Some(&"\"true\"".to_string()) == self.attributes.get("aria-hidden") {
return output;
} else if self.text_node {
output.push(OutputType::Text(handle_escaped(&self.tag_name.clone())));
output.push(OutputType::Text(handle_escaped(&self.tag_name.clone(), REPLACE.to_vec(), false)));
return output;
} else if self.tag_name == "script" || self.tag_name == "style" {
//ignore script and style tags
@@ -98,7 +118,15 @@ impl Node {
output.push(OutputType::Text("-".to_string()));
} else if let Some(href) = self.attributes.get("href") {
link = true;
output.push(OutputType::StartLink(href.to_string()));
//check if href is ddg link that fucks us over in lite.duckduckgo.com
// //duckduckgo.com/l/?uddg=https%3A%2F%2Fwww.merriam%2Dwebster.com%2Fdictionary%2Ftest&amp;rut=f86942690bea49b300b8ae8d470dbbe18ad217aded1750804e3f33a95da21cf2
let href = if href.starts_with("\"//duckduckgo.com/l/?uddg=") {
//todo: only take from &amp onward
"\"".to_string() + &handle_escaped(&href.chars().skip(26).collect::<String>().split("&amp;").next().unwrap(), URL_REPLACE.to_vec(), false) + "\""
} else {
href.to_string()
};
output.push(OutputType::StartLink(href));
} else if let Some(indent) = self.attributes.get("indent") {
//non-standard indent attribute, basically just to support HN
let indent = remove_quotes(indent.to_string());
@@ -109,17 +137,25 @@ impl Node {
if let Some(name) = self.attributes.get("name") {
//unwrap_or is painful so compiler suggested map_or
let input_type = remove_quotes(self.attributes.get("type").map_or("\"text\"".to_string(), |v| v.to_string()));
if input_type == "text" || input_type == "search" {
output.push(OutputType::TextInput(remove_quotes(name.to_string())));
if input_type == "text" || input_type == "search" || input_type == "password" || input_type == "hidden" {
let default_value = remove_quotes(self.attributes.get("value").map_or(String::new(), |v| v.to_string()));
output.push(OutputType::TextInput(remove_quotes(name.to_string()), default_value));
}
}
} else if self.tag_name == "button" {
if Some(&"\"submit\"".to_string()) == self.attributes.get("type") {
//we only care about submit buttons with names since we need to send that on form POST submit
if let Some(name) = self.attributes.get("name") {
let default_value = remove_quotes(self.attributes.get("value").map_or(String::new(), |v| v.to_string()));
output.push(OutputType::TextInput(remove_quotes(name.to_string()), default_value));
}
}
} else if self.tag_name == "form" {
if let Some(action) = self.attributes.get("action") {
let action = self.attributes.get("action");
let method = if let Some(m) = self.attributes.get("method") {
let m = remove_quotes(m.to_string()).to_lowercase();
match m.as_str() {
//todo: POST is currently not implemented. maybe later
//"post" => Some(FormSubmitMethod::Post),
"post" => Some(FormSubmitMethod::Post),
"get" => Some(FormSubmitMethod::Get),
_ => None,
}
@@ -128,18 +164,18 @@ impl Node {
};
if let Some(method) = method {
form = Some(Form {
action: remove_quotes(action.to_string()),
//wikipedia puts &amp; in the action url??? is that how its supposed to be? do I need to worry about href?
action: if let Some(action) = action { Some(handle_escaped(&remove_quotes(action.to_string()), REPLACE.to_vec(), false)) } else { None },
method,
input_names: Vec::new(),
});
}
}
}
for c in &self.children {
let children_output = c.to_output();
if form.is_some() {
for cc in &children_output {
if let OutputType::TextInput(name) = cc {
if let OutputType::TextInput(name, _) = cc {
input_names.push(name.to_string());
}
}
@@ -186,6 +222,7 @@ pub fn parse(xml_string: &str) -> Vec<Box<Node>> {
let mut attribute_name = String::new();
let mut recording_attribute_value = false;
let mut in_string = false;
let mut quote_type = None;
let mut current_node: Option<Node> = None;
loop {
let c = chars.next();
@@ -241,7 +278,7 @@ pub fn parse(xml_string: &str) -> Vec<Box<Node>> {
}
} else if (c == ' ' || c == '\n') && recording_tag_name && !n.text_node {
recording_tag_name = false;
} else if c == '>' || (c == '/' && chars.peek().unwrap_or(&' ') == &'>') || (n.text_node && chars.peek().unwrap_or(&' ') == &'<') {
} else if (c == '>' || (c == '/' && chars.peek().unwrap_or(&' ') == &'>') || (n.text_node && chars.peek().unwrap_or(&' ') == &'<')) && (!in_string || quote_type == Some(c)) {
if n.text_node {
n.tag_name += &c.to_string();
}
@@ -271,8 +308,12 @@ pub fn parse(xml_string: &str) -> Vec<Box<Node>> {
}
attribute_name = String::new();
} else if recording_attribute_value {
if c == '"' {
if (c == '"' || c == '\'') && (quote_type == Some(c) || quote_type.is_none()) {
in_string = *n.attributes.get(&attribute_name).unwrap() == "";
quote_type = Some(c);
if !in_string {
quote_type = None;
}
}
n.attributes.entry(attribute_name.clone()).and_modify(|s| *s += &c.to_string());
} else if c == '=' {
@@ -332,6 +373,7 @@ fn test_xml_parse() {
assert!(nodes[0].children[1].children[0].tag_name == "lorem ipsum");
assert!(nodes[0].children[2].tag_name == " !!! no way");
assert!(nodes[1].tag_name == "input");
println!("{}", nodes[1].attributes.get("name").unwrap());
assert!(nodes[1].attributes.get("name").unwrap() == "\"in put\"");
assert!(nodes[2].tag_name == "div");
assert!(nodes[2].children.len() == 2);
@@ -406,12 +448,20 @@ fn test_form_parse_and_output() {
//
}
#[test]
fn test_strings_again() {
let nodes = parse("<span data-value='woah\"cheeseburgers\"'>Nice</span>");
assert!(nodes[0].attributes.get("data-value").unwrap() == "'woah\"cheeseburgers\"'");
let nodes = parse("<span data-value=\"woah! ' cheeseburgers'\">Nice</span>");
assert!(nodes[0].attributes.get("data-value").unwrap() == "\"woah! ' cheeseburgers'\"");
}
/*#[test]
#[test]
fn test_real() {
use std::fs::read_to_string;
let nodes = parse(&read_to_string("./real_tests/wikipedia.html").unwrap());
println!("{:#?}", nodes[1].children);
//println!("{:#?}", nodes);
println!("{:?}", nodes[1].children[1].to_output());
println!("{}", nodes[1].children[1].tag_name);
}*/
//println!("{}", nodes[12323233].children[1].tag_name);
}