improve curl

Ferenc Szontágh

2025-04-18 7934a339e9b2319e0234e8f2ca5d952eff243c05

improve curl

 CMakeLists.txt

@@ -156,7 +156,7 @@
endif()

# Plugin modules options
option(BUILD_MODULE_CURL "Enable building CurlModule" OFF)
option(BUILD_MODULE_CURL "Enable building CurlModule" ON)

if (BUILD_MODULE_CURL)
 add_subdirectory(Modules/CurlModule)

 Modules/CurlModule/README.md

New file
@@ -0,0 +1,80 @@
# CurlModule

This module provides HTTP GET and POST functionality via libcurl in VoidScript.

## Functions

### curlGet
`curlGet(url [, options]) -> string`

- `url` (string): The HTTP/HTTPS URL to request.
- `options` (object, optional): Configuration object with fields:
  - `timeout` (int or float): Maximum time in seconds to wait for the request.
  - `follow_redirects` (bool): Whether to follow HTTP redirects (default: false).
  - `headers` (object): Custom HTTP headers as key/value pairs. Keys must be valid identifiers (no hyphens).

Returns the response body as a string.

### curlPost
`curlPost(url, data [, options]) -> string`

- `url` (string): The HTTP/HTTPS URL to send the POST request.
- `data` (string): Request body (e.g., a JSON-encoded payload).
- `options` (object, optional): Same as for `curlGet`. If `headers` does not include a `Content-Type`,
  `Content-Type: application/json` is automatically added.

Returns the response body as a string.

## Examples

Basic GET:
```vs
string $resp = curlGet("https://jsonplaceholder.typicode.com/todos/1");
printnl($resp);
```

GET with options:
```vs
string $resp = curlGet("https://jsonplaceholder.typicode.com/todos/1", {
    timeout: 5,
    follow_redirects: true,
    headers: {
        Accept: "application/json"
    }
});
printnl($resp);
```

Basic POST (JSON payload):
```vs
object $payload = { string title: "foo", string body: "bar", int userId: 1 };
string $json = json_encode($payload);
string $resp = curlPost("https://jsonplaceholder.typicode.com/posts", $json);
printnl($resp);
```

POST with options:
```vs
object $payload = { string name: "Alice", int age: 30 };
string $json = json_encode($payload);
string $resp = curlPost("https://example.com/api/users", $json, {
    timeout: 10,
    follow_redirects: true,
    headers: {
        X_Test: "CustomValue"
    }
});
printnl($resp);
```

## Integration

Ensure the `CurlModule` is registered before running scripts:
```cpp
Modules::ModuleManager::instance().addModule(
    std::make_unique<Modules::CurlModule>()
);
Modules::ModuleManager::instance().registerAll();
```

Place this file alongside `CMakeLists.txt` and `src/` in the `Modules/CurlModule/` folder.

 Modules/CurlModule/src/CurlModule.cpp

@@ -6,6 +6,7 @@
#include <stdexcept>
#include <string>
#include <vector>
#include <algorithm>

// Callback for libcurl to write received data into a std::string
static size_t write_callback(void* ptr, size_t size, size_t nmemb, void* userdata) {
@@ -29,58 +30,181 @@


Symbols::Value Modules::CurlModule::curlPost(const std::vector<Symbols::Value>& args) {
    if (args.size() != 2) {
        throw std::runtime_error("curlPost: missing URL and data arguments");
    // curlPost: url, data [, options]
    if (args.size() < 2 || args.size() > 3) {
        throw std::runtime_error("curlPost: expects url, data, and optional options object");
    }
    std::string url  = Symbols::Value::to_string(args[0]);
    std::string data = Symbols::Value::to_string(args[1]);

    struct curl_slist *headers = nullptr;
    long timeoutSec = 0;
    bool follow = false;
    bool haveContentType = false;
    if (args.size() == 3) {
        using namespace Symbols;
        if (args[2].getType() != Variables::Type::OBJECT) {
            throw std::runtime_error("curlPost: options must be object");
        }
        const auto & obj = std::get<Value::ObjectMap>(args[2].get());
        for (const auto & kv : obj) {
            const std::string & key = kv.first;
            const Value & v = kv.second;
            if (key == "timeout") {
                using namespace Variables;
                switch (v.getType()) {
                    case Type::INTEGER:
                        timeoutSec = v.get<int>(); break;
                    case Type::DOUBLE:
                        timeoutSec = static_cast<long>(v.get<double>()); break;
                    case Type::FLOAT:
                        timeoutSec = static_cast<long>(v.get<float>()); break;
                    default:
                        throw std::runtime_error("curlPost: timeout must be number");
                }
            } else if (key == "follow_redirects") {
                if (v.getType() != Variables::Type::BOOLEAN) {
                    throw std::runtime_error("curlPost: follow_redirects must be boolean");
                }
                follow = v.get<bool>();
            } else if (key == "headers") {
                if (v.getType() != Variables::Type::OBJECT) {
                    throw std::runtime_error("curlPost: headers must be object");
                }
                const auto & hobj = std::get<Value::ObjectMap>(v.get());
                for (const auto & hk : hobj) {
                    if (hk.second.getType() != Variables::Type::STRING) {
                        throw std::runtime_error("curlPost: header values must be string");
                    }
                    std::string hdr = hk.first + ": " + hk.second.get<std::string>();
                    std::string lower = hk.first;
                    std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower);
                    if (lower == "content-type") {
                        haveContentType = true;
                    }
                    headers = curl_slist_append(headers, hdr.c_str());
                }
            } else {
                throw std::runtime_error("curlPost: unknown option '" + key + "'");
            }
        }
    }
    CURL * curl = curl_easy_init();
    if (!curl) {
        if (headers) curl_slist_free_all(headers);
        throw std::runtime_error("curl: failed to initialize");
    }

    std::string response;
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    if (timeoutSec > 0) {
        curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeoutSec);
    }
    if (follow) {
        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    }
    if (!haveContentType) {
        headers = curl_slist_append(headers, "Content-Type: application/json");
    }
    if (headers) {
        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
    }
    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.c_str());
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);

    CURLcode res = curl_easy_perform(curl);
    if (headers) curl_slist_free_all(headers);
    if (res != CURLE_OK) {
        std::string error = curl_easy_strerror(res);
        curl_easy_cleanup(curl);
        throw std::runtime_error("curl: request failed: " + error);
    }

    curl_easy_cleanup(curl);
    return Symbols::Value(response);
}

Symbols::Value Modules::CurlModule::curlGet(const std::vector<Symbols::Value>& args) {
    if (args.size() != 1) {
        throw std::runtime_error("curlGet: missing URL argument");
    // curlGet: url [, options]
    if (args.size() < 1 || args.size() > 2) {
        throw std::runtime_error("curlGet: expects url and optional options object");
    }

    std::string url = Symbols::Value::to_string(args[0]);

    // parse options
    struct curl_slist *headers = nullptr;
    long timeoutSec = 0;
    bool follow = false;
    if (args.size() == 2) {
        using namespace Symbols;
        if (args[1].getType() != Variables::Type::OBJECT) {
            throw std::runtime_error("curlGet: options must be object");
        }
        const auto & obj = std::get<Value::ObjectMap>(args[1].get());
        for (const auto & kv : obj) {
            const std::string & key = kv.first;
            const Value & v = kv.second;
            if (key == "timeout") {
                using namespace Variables;
                switch (v.getType()) {
                    case Type::INTEGER:
                        timeoutSec = v.get<int>();
                        break;
                    case Type::DOUBLE:
                        timeoutSec = static_cast<long>(v.get<double>());
                        break;
                    case Type::FLOAT:
                        timeoutSec = static_cast<long>(v.get<float>());
                        break;
                    default:
                        throw std::runtime_error("curlGet: timeout must be number");
                }
            } else if (key == "follow_redirects") {
                if (v.getType() != Symbols::Variables::Type::BOOLEAN) {
                    throw std::runtime_error("curlGet: follow_redirects must be boolean");
                }
                follow = v.get<bool>();
            } else if (key == "headers") {
                if (v.getType() != Symbols::Variables::Type::OBJECT) {
                    throw std::runtime_error("curlGet: headers must be object");
                }
                const auto & hobj = std::get<Value::ObjectMap>(v.get());
                for (const auto & hk : hobj) {
                    if (hk.second.getType() != Symbols::Variables::Type::STRING) {
                        throw std::runtime_error("curlGet: header values must be string");
                    }
                    std::string line = hk.first + ": " + hk.second.get<std::string>();
                    headers = curl_slist_append(headers, line.c_str());
                }
            } else {
                throw std::runtime_error("curlGet: unknown option '" + key + "'");
            }
        }
    }
    // initialize handle
    CURL * curl = curl_easy_init();
    if (!curl) {
        if (headers) curl_slist_free_all(headers);
        throw std::runtime_error("curl: failed to initialize");
    }

    std::string response;
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    if (timeoutSec > 0) {
        curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeoutSec);
    }
    if (follow) {
        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    }
    if (headers) {
        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
    }
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);

    CURLcode res = curl_easy_perform(curl);
    if (headers) {
        curl_slist_free_all(headers);
    }
    if (res != CURLE_OK) {
        std::string error = curl_easy_strerror(res);
        curl_easy_cleanup(curl);
        throw std::runtime_error("curl: request failed: " + error);
    }

    curl_easy_cleanup(curl);
    return Symbols::Value(response);
}

 Modules/CurlModule/src/CurlModule.hpp

@@ -16,12 +16,20 @@
    void registerModule() override;
    
    /**
     * @brief Perform HTTP GET: curlGet(url)
     * @brief Perform HTTP GET: curlGet(url [, options])
     * options is an object with optional fields:
     *   timeout (int or double seconds),
     *   follow_redirects (bool),
     *   headers (object mapping header names to values)
     */
    Symbols::Value curlGet(const std::vector<Symbols::Value>& args);
    
    /**
     * @brief Perform HTTP POST: curlPost(url, data)
     * @brief Perform HTTP POST: curlPost(url, data [, options])
     * options is an object with optional fields:
     *   timeout (int or double seconds),
     *   follow_redirects (bool),
     *   headers (object mapping header names to values)
     */
    Symbols::Value curlPost(const std::vector<Symbols::Value>& args);
};

 src/Modules/JsonModule.hpp

New file
@@ -0,0 +1,239 @@
// JsonModule.hpp
#ifndef MODULES_JSONMODULE_HPP
#define MODULES_JSONMODULE_HPP

#include <string>
#include <map>
#include <variant>
#include <cctype>
#include <stdexcept>
#include <sstream>
#include "BaseModule.hpp"
#include "ModuleManager.hpp"
#include "Symbols/Value.hpp"
#include "Symbols/VariableTypes.hpp"

namespace Modules {

/**
 * @brief Module providing JSON encode/decode functions.
 *   json_encode(value) -> string
 *   json_decode(string) -> object/value
 */
class JsonModule : public BaseModule {
  public:
    void registerModule() override {
        auto &mgr = ModuleManager::instance();
        // json_encode: serialize a Value to JSON string
        mgr.registerFunction("json_encode", [](const std::vector<Symbols::Value> &args) {
            using namespace Symbols;
            if (args.size() != 1) {
                throw std::runtime_error("json_encode expects 1 argument");
            }
            // forward to encoder
            std::function<std::string(const Value &)> encode;
            encode = [&](const Value &v) -> std::string {
                const auto &var = v.get();
                return std::visit(
                    [&](auto &&x) -> std::string {
                        using T = std::decay_t<decltype(x)>;
                        if constexpr (std::is_same_v<T, bool>) {
                            return x ? "true" : "false";
                        } else if constexpr (std::is_same_v<T, int> || std::is_same_v<T, double> || std::is_same_v<T, float>) {
                            return std::to_string(x);
                        } else if constexpr (std::is_same_v<T, std::string>) {
                            // escape string
                            std::string out = "\"";
                            for (char c : x) {
                                switch (c) {
                                    case '"': out += "\\\""; break;
                                    case '\\': out += "\\\\"; break;
                                    case '\b': out += "\\b"; break;
                                    case '\f': out += "\\f"; break;
                                    case '\n': out += "\\n"; break;
                                    case '\r': out += "\\r"; break;
                                    case '\t': out += "\\t"; break;
                                    default:
                                        if (static_cast<unsigned char>(c) < 0x20) {
                                            // control character
                                            char buf[7];
                                            std::snprintf(buf, sizeof(buf), "\\u%04x", c);
                                            out += buf;
                                        } else {
                                            out += c;
                                        }
                                }
                            }
                            out += "\"";
                            return out;
                        } else if constexpr (std::is_same_v<T, Value::ObjectMap>) {
                            std::string out = "{";
                            bool first = true;
                            for (const auto &kv : x) {
                                if (!first) out += ",";
                                first = false;
                                // key
                                out += '"';
                                // escape key string
                                for (char c : kv.first) {
                                    switch (c) {
                                        case '"': out += "\\\""; break;
                                        case '\\': out += "\\\\"; break;
                                        case '\b': out += "\\b"; break;
                                        case '\f': out += "\\f"; break;
                                        case '\n': out += "\\n"; break;
                                        case '\r': out += "\\r"; break;
                                        case '\t': out += "\\t"; break;
                                        default:
                                            if (static_cast<unsigned char>(c) < 0x20) {
                                                char buf[7];
                                                std::snprintf(buf, sizeof(buf), "\\u%04x", c);
                                                out += buf;
                                            } else {
                                                out += c;
                                            }
                                    }
                                }
                                out += '"';
                                out += ':';
                                out += encode(kv.second);
                            }
                            out += "}";
                            return out;
                        } else {
                            return "null";
                        }
                    },
                    var);
            };
            std::string result = encode(args[0]);
            return Symbols::Value(result);
        });
        // json_decode: parse JSON string to Value (object/value)
        mgr.registerFunction("json_decode", [](const std::vector<Symbols::Value> &args) {
            using namespace Symbols;
            if (args.size() != 1) {
                throw std::runtime_error("json_decode expects 1 argument");
            }
            if (args[0].getType() != Variables::Type::STRING) {
                throw std::runtime_error("json_decode expects a JSON string");
            }
            const std::string s = args[0].get<std::string>();
            struct Parser {
                const std::string &s;
                size_t pos = 0;
                Parser(const std::string &str) : s(str), pos(0) {}
                void skip() {
                    while (pos < s.size() && std::isspace(static_cast<unsigned char>(s[pos]))) pos++;
                }
                std::string parseString() {
                    skip();
                    if (s[pos] != '"') throw std::runtime_error("Invalid JSON string");
                    pos++;
                    std::string out;
                    while (pos < s.size()) {
                        char c = s[pos++];
                        if (c == '"') break;
                        if (c == '\\') {
                            if (pos >= s.size()) break;
                            char e = s[pos++];
                            switch (e) {
                                case '"': out += '"'; break;
                                case '\\': out += '\\'; break;
                                case '/': out += '/'; break;
                                case 'b': out += '\b'; break;
                                case 'f': out += '\f'; break;
                                case 'n': out += '\n'; break;
                                case 'r': out += '\r'; break;
                                case 't': out += '\t'; break;
                                default: out += e; break;
                            }
                        } else {
                            out += c;
                        }
                    }
                    return out;
                }
                Value parseNumber() {
                    skip();
                    size_t start = pos;
                    if (s[pos] == '-') pos++;
                    while (pos < s.size() && std::isdigit(static_cast<unsigned char>(s[pos]))) pos++;
                    bool isDouble = false;
                    if (pos < s.size() && s[pos] == '.') {
                        isDouble = true;
                        pos++;
                        while (pos < s.size() && std::isdigit(static_cast<unsigned char>(s[pos]))) pos++;
                    }
                    std::string num = s.substr(start, pos - start);
                    try {
                        if (isDouble) {
                            return Value(std::stod(num));
                        }
                        return Value(std::stoi(num));
                    } catch (...) {
                        throw std::runtime_error("Invalid JSON number: " + num);
                    }
                }
                Value parseBool() {
                    skip();
                    if (s.compare(pos, 4, "true") == 0) {
                        pos += 4;
                        return Value(true);
                    } else if (s.compare(pos, 5, "false") == 0) {
                        pos += 5;
                        return Value(false);
                    }
                    throw std::runtime_error("Invalid JSON boolean");
                }
                Value parseNull() {
                    skip();
                    if (s.compare(pos, 4, "null") == 0) {
                        pos += 4;
                        return Value::makeNull();
                    }
                    throw std::runtime_error("Invalid JSON null");
                }
                Value parseObject() {
                    skip();
                    if (s[pos] != '{') throw std::runtime_error("Invalid JSON object");
                    pos++;
                    skip();
                    Value::ObjectMap obj;
                    if (s[pos] == '}') { pos++; return Value(obj); }
                    while (pos < s.size()) {
                        skip();
                        std::string key = parseString();
                        skip();
                        if (s[pos] != ':') throw std::runtime_error("Expected ':' in object");
                        pos++;
                        skip();
                        Value val = parseValue();
                        obj.emplace(key, val);
                        skip();
                        if (s[pos] == ',') { pos++; continue; }
                        if (s[pos] == '}') { pos++; break; }
                        throw std::runtime_error("Expected ',' or '}' in object");
                    }
                    return Value(obj);
                }
                Value parseValue() {
                    skip();
                    if (pos >= s.size()) throw std::runtime_error("Empty JSON");
                    char c = s[pos];
                    if (c == '{') return parseObject();
                    if (c == '"') { std::string str = parseString(); return Value(str); }
                    if (c == 't' || c == 'f') return parseBool();
                    if (c == 'n') return parseNull();
                    if (c == '-' || std::isdigit(static_cast<unsigned char>(c))) return parseNumber();
                    throw std::runtime_error(std::string("Invalid JSON value at pos ") + std::to_string(pos));
                }
            } parser(s);
            Value result = parser.parseValue();
            return result;
        });
    }
};

} // namespace Modules
#endif // MODULES_JSONMODULE_HPP

 src/VoidScript.hpp

@@ -11,6 +11,7 @@
#include "Modules/PrintModule.hpp"
#include "Modules/TypeofModule.hpp"
#include "Modules/FileModule.hpp"
#include "Modules/JsonModule.hpp"
#include "Parser/Parser.hpp"

class VoidScript {
@@ -60,6 +61,8 @@
        Modules::ModuleManager::instance().addModule(std::make_unique<Modules::TypeofModule>());
        // file I/O builtin
        Modules::ModuleManager::instance().addModule(std::make_unique<Modules::FileModule>());
        // JSON encode/decode builtin
        Modules::ModuleManager::instance().addModule(std::make_unique<Modules::JsonModule>());
        this->files.emplace(this->files.begin(), file);

        lexer->setKeyWords(Parser::Parser::keywords);

 test_scripts/curl_json.vs

New file
@@ -0,0 +1,38 @@
# CURL + JSON Feature Test
# GET request
string $url = "https://jsonplaceholder.typicode.com/todos/1";
string $resp = curlGet($url);
printnl("GET raw: ", $resp);
object $data = json_decode($resp);
printnl("ID: ", $data->id, " Title: ", $data->title, " Completed: ", $data->completed);

# POST request
string $postUrl = "https://jsonplaceholder.typicode.com/posts";
object $payload = {
    string title: "foo",
    string body: "bar",
    int userId: 1
};
string $postData = json_encode($payload);
printnl("POST data: ", $postData);
string $postResp = curlPost($postUrl, $postData);
printnl("POST raw: ", $postResp);
object $postJson = json_decode($postResp);
// The response from JSONPlaceholder includes only the new id
printnl("POST ID: ", $postJson->id);
// --- GET with options ---
printnl("GET with options (timeout=5, follow_redirects, Accept header):");
string $optGet = curlGet($url, {
    timeout: 5,
    follow_redirects: true,
    headers: { Accept: "application/json" }
});
printnl($optGet);
// --- POST with options ---
printnl("POST with options (timeout=5, follow_redirects, X_Test header):");
string $optPost = curlPost($postUrl, $postData, {
    timeout: 5,
    follow_redirects: true,
    headers: { X_Test: "CustomValue" }
});
printnl($optPost);

 test_scripts/json.vs

New file
@@ -0,0 +1,49 @@
# JSON Encode/Decode Feature Test
# Define an object with nested data
object $user = {
    string name: "Alice",
    int age: 30,
    boolean active: true,
    object prefs: {
        string theme: "dark",
        boolean notifications: false
    }
};

// Encode to JSON string
string $json = json_encode($user);
printnl("Encoded JSON: ", $json);

// Decode back to object
object $parsed = json_decode($json);
// Re-encode to verify round-trip
string $json2 = json_encode($parsed);
printnl("Re-encoded JSON: ", $json2);
// --- Simple value tests ---
// Integer
int $num = 42;
string $num_json = json_encode($num);
printnl("Encoded integer: ", $num_json);
int $num_decoded = json_decode($num_json);
printnl("Decoded integer: ", $num_decoded);

// String
string $str = "Hello, VoidScript!";
string $str_json = json_encode($str);
printnl("Encoded string: ", $str_json);
string $str_decoded = json_decode($str_json);
printnl("Decoded string: ", $str_decoded);

// Boolean
boolean $flag = true;
string $flag_json = json_encode($flag);
printnl("Encoded boolean: ", $flag_json);
boolean $flag_decoded = json_decode($flag_json);
printnl("Decoded boolean: ", $flag_decoded);

// Double
double $pi = 3.14159;
string $pi_json = json_encode($pi);
printnl("Encoded double: ", $pi_json);
double $pi_decoded = json_decode($pi_json);
printnl("Decoded double: ", $pi_decoded);