doc: github pilot docs

src/tokenizer/tokenizer.c

@@ -6,12 +6,23 @@
 /*   By: qmennen <qmennen@student.codam.nl>         +#+  +:+       +#+        */
 /*                                                +#+#+#+#+#+   +#+           */
 /*   Created: 2025/02/04 16:07:58 by qmennen           #+#    #+#             */
-/*   Updated: 2025/02/04 20:52:58 by qmennen          ###   ########.fr       */
+/*   Updated: 2025/02/04 20:57:25 by qmennen          ###   ########.fr       */
 /*                                                                            */
 /* ************************************************************************** */
 
 #include "minishell.h"
 
+/**
+ * @brief Parses the input from the lexer and returns a list of tokens.
+ *
+ * This function continuously retrieves the next token from the lexer and adds
+ * it to a linked list until an end-of-file (EOF) or error token is encountered.
+ * The list of tokens is then returned.
+ *
+ * @param lexer A pointer to the lexer structure containing
+ * 				the input to be parsed.
+ * @return A linked list of tokens parsed from the input.
+ */
 t_list	*ft_parse_input(t_lexer *lexer)
 {
 	t_list			*list;
@@ -29,6 +40,25 @@ t_list	*ft_parse_input(t_lexer *lexer)
 	return (list);
 }
 
+/**
+ * @brief Retrieves the next token from the lexer.
+ *
+ * This function reads the next token from the lexer, skipping any whitespace
+ * characters. It handles different types of tokens such as end-of-file (EOF),
+ * special characters ('<', '>', '|'), printable characters, and errors.
+ *
+ * @param lexer A pointer to the lexer structure.
+ * @return A pointer to the newly created token.
+ *
+ * The function performs the following steps:
+ * 1. Skips any whitespace characters.
+ * 2. Checks the current character in the lexer:
+ *    - If it is the end-of-file character ('\0'), creates an EOF token.
+ *    - If it is a special character ('<', '>', '|'), parses 
+ * 		the token accordingly.
+ *    - If it is a printable character, reads the word and creates a word token.
+ *    - Otherwise, creates an error token.
+ */
 t_token	*ft_token_next(t_lexer *lexer)
 {
 	t_token	*token;